framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,3,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1,0.023658665517965954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,3,0.023290666441122692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,7,0.025600001215934753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,15,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,15,0.023423999547958374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,31,0.025573333104451496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,31,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,63,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,7,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,127,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,127,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,255,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,255,0.02478400121132533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,511,0.03756266583998998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1023,0.05624000231424967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,2047,0.0974826713403066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,2047,0.08516800403594971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,4095,0.17117865880330405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1023,0.050016000866889954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,4095,0.1465120017528534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,511,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,511,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1023,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,2047,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,2047,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,4095,0.020410666863123577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,4095,0.015402667224407196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,8191,0.021712000171343487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,8191,0.02037866661945979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,16383,0.03548266738653183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,3,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,15,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,127,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1023,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1023,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,2047,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,2047,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,511,0.03160000095764796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,4095,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,8191,0.02956799914439519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,16383,0.056261335810025535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,8191,0.03403199960788091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,16383,0.049925332268079124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1,0.037952000896135964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,3,0.0399893323580424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,16383,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,4095,0.021562665700912476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,7,0.0377866675456365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,15,0.03994133323431015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,7,0.04101333270470301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,15,0.03770666569471359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,31,0.04081066697835922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,31,0.037861332297325134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1,0.039605334401130676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,63,0.03793599953254064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,127,0.03994666785001755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,127,0.03770133356253306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,255,0.04185600082079569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,255,0.038106667498747505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,511,0.06221333146095276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,511,0.05614933371543884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1023,0.09715200463930766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1023,0.08540800213813782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,2047,0.17129600048065186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,2047,0.1485973298549652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,63,0.04163199911514918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,3,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,3,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,7,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,7,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,15,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,63,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,63,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,127,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,255,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,255,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,511,0.01351999988158544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,511,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1023,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1023,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,2047,0.023717333873112995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,2047,0.023306667804718018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,4095,0.03766933331886927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,4095,0.031658666829268135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,8191,0.05930666625499725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,8191,0.05356266597906748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,16383,0.10156266887982686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,16383,0.08797333637873332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1,0.07046933472156525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1,0.06420266628265381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,3,0.07049066821734111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,3,0.06431999802589417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,7,0.07042133311430614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,7,0.06470933556556702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,15,0.07029866675535838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,15,0.06444799900054932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,31,0.06420266628265381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,63,0.0706826647122701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,63,0.06447466711203258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,127,0.07064533233642578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,127,0.06417066852251689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,255,0.07213866710662842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,255,0.06619200110435486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,511,0.11162666479746501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,511,0.09759466846783955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1023,0.18109333515167236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1023,0.15690132975578308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,31,0.07082666456699371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,1,0.11980266372362773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,3,0.03756266583998998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,3,0.11972266435623169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,7,0.132042666276296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,7,0.11957333485285442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,15,0.13190933068593344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,15,0.11776533722877502
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,31,0.13198399543762207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,31,0.11771733562151591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,63,0.1330773333708445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,63,0.11924266815185547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,1,0.13249066472053528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,127,0.1299199958642324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,127,0.11982933680216472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,255,0.1302186648050944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,255,0.1200320025285085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,1,0.2528640031814575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,1,0.23011734088261923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,3,0.2531840006510417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,3,0.2286240061124166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,7,0.25381332635879517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,7,0.2297066648801168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,15,0.2529546618461609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,15,0.23031999667485556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,31,0.2530133326848348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,3,0.1327839990456899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,31,0.22830933332443237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,63,0.25382399559020996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,63,0.23035732905069986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,127,0.2474773327509562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,3,0.014426667243242264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,7,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,7,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,15,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,15,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,31,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,31,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,63,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1,0.013925333817799887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,127,0.014346666634082794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,3,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,255,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,255,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,511,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,511,0.016623999923467636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1023,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1023,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,2047,0.039461334546407066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,63,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,2047,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,127,0.23004267613093057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,127,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,4095,0.05388266841570536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,8191,0.10332799951235454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,8191,0.08904533584912618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,16383,0.18741865952809653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,16383,0.15876799821853638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,1,0.49535465240478516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,4095,0.06070399781068166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,3,0.49514134724934894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,7,0.5010293324788412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,1,0.4456533193588257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,7,0.4472106695175171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,15,0.4949066638946533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,15,0.446725328763326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,31,0.494917352994283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,31,0.4458506504694621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,63,0.49486398696899414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,63,0.4477599859237671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,3,0.44580801328023273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,1,0.9764426549275717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,1,0.8793599605560303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,3,0.9778506755828857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,3,0.8785546620686849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,7,0.97707732518514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,7,0.879093329111735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,15,0.9763893286387125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,15,0.8790826797485352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,3,0.017583999782800674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,31,0.9767946402231852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,31,0.8797919750213623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,7,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,7,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,15,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,15,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,31,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,31,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,63,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,63,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,127,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,127,0.017525333911180496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,255,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,255,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,511,0.023018665611743927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,511,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1023,0.03543466577927271
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1023,0.029322666426499683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,3,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,2047,0.04990399877230326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,4095,0.09353599945704143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,4095,0.08107199768225352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,8191,0.16796799500783285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,8191,0.14224533240000406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,3,0.02937600016593933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,3,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,7,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,7,0.02513599892457326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,15,0.029135999580224354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,15,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,31,0.029232000311215717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,31,0.02536533276240031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,2047,0.056101332108179726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,63,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,127,0.02897600084543228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,127,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,255,0.031957333286603294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,255,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,511,0.05180266499519348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,511,0.03749866783618927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1023,0.0892639954884847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1023,0.054010664423306785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,2047,0.16885866721471152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,4095,0.32038400570551556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,63,0.029605334003766377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,4095,0.16695467631022134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,2047,0.09464533130327861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,31,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,511,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1023,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1023,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,2047,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,2047,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,4095,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1,0.017530667285124462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,4095,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,8191,0.03509866694609324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,8191,0.021503999829292297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,16383,0.056176001826922096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,16383,0.0341333324710528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,31,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,63,0.011727999895811081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,127,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,255,0.011589333415031433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,511,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1023,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,2047,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,4095,0.035743998984495796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,4095,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,8191,0.055813332398732506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,8191,0.035504000882307686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,16383,0.09715732932090759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1,0.040362666050593056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,3,0.04780266682306925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,3,0.03986666599909464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,7,0.04808000226815542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,7,0.04033066580692927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,15,0.04808533191680908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,15,0.03972266614437103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,16383,0.056032001972198486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1,0.048528000712394714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,63,0.04781866570313772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,63,0.03991466760635376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,127,0.049925332268079124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,127,0.03955200066169103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,255,0.05183466772238413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,255,0.04154133299986521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,31,0.04794666667779287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,31,0.040037333965301514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,511,0.06018133461475372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1023,0.16615466276804605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,2047,0.31674132744471234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,2047,0.16886399189631143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,3,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,3,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,7,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,511,0.09261332949002583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,15,0.01250133290886879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,31,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1023,0.09506133198738098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,31,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,63,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,63,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,127,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,127,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,255,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,255,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,511,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,511,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,7,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1023,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,2047,0.03763733307520548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,2047,0.023669332265853882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,4095,0.05875733494758606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,4095,0.038032000263532005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,8191,0.10057066877683003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,8191,0.05840000013510386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,16383,0.18357867002487183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1,0.0869760016600291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1,0.07067733506361644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1023,0.01887999971707662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,3,0.08639466762542725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,3,0.0709386666615804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,7,0.08662399649620056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,7,0.07076799869537354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,15,0.08661333719889323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,15,0.07085333267847697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,31,0.08644800384839375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,31,0.07087466617425282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,16383,0.09948800007502238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,63,0.0708000014225642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,127,0.0869813362757365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,127,0.07088000078996022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,255,0.090938667456309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,255,0.07241066793600719
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,511,0.16808533668518066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,511,0.10751466949780782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1023,0.31329067548116046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,63,0.08660800258318584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1023,0.17550933361053467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,1,0.1604639987150828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,1,0.1320853332678477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,3,0.16082666317621866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,15,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,7,0.1607093314329783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,7,0.1328373352686564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,15,0.160863995552063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,15,0.13330666224161783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,31,0.16076266765594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,31,0.13235732913017273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,63,0.16080000003178915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,63,0.13243732849756876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,127,0.1625386675198873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,3,0.1321386694908142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,255,0.16884799798329672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,255,0.13160533706347147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,1,0.2521013418833415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,3,0.30990399916966754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,3,0.25284266471862793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,7,0.3102346658706665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,127,0.12982400258382162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,15,0.3094506661097209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,15,0.2528853416442871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,31,0.3109546701113383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,1,0.3109920024871826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,31,0.2534346580505371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,63,0.31030933062235516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,127,0.312608003616333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,3,0.015637333194414776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,3,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,7,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,7,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,15,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,63,0.2531573375066121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,31,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,31,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,63,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,63,0.014709333578745524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,127,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,127,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,255,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,255,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,511,0.019402666638294857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,511,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1023,0.03412266572316488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1023,0.021221332252025604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,2047,0.058549334605534874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,2047,0.042122667034467064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,4095,0.10124267141024272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,4095,0.06051200131575266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,15,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,8191,0.18452266852060953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,8191,0.1016480028629303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,7,0.252895991007487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,16383,0.3511626720428467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,1,0.6071466604868571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,1,0.49235733350118
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,3,0.607258677482605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,3,0.4915626843770345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,16383,0.18305599689483643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,7,0.6071199973424276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,7,0.49208001295725506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,15,0.6070826848347982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,15,0.49483732382456463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,127,0.24723732471466064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,31,0.6071946620941162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,31,0.4910026788711548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,63,0.6073600053787231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,1,0.967029333114624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,3,1.1988266309102376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,63,0.4920479853947957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,3,0.967082659403483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,1,1.198896010716756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,7,1.19868270556132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,7,0.9669919808705648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,15,1.1987573305765789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,15,0.9666879971822103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,3,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,31,1.198789358139038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,7,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,7,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,15,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,15,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,31,0.019487999379634857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1,0.019968000551064808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,63,0.020901332298914593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,63,0.01876266673207283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,127,0.019402666638294857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,127,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,255,0.0195573332409064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,255,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,511,0.033701332906881966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,31,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,511,0.023344000180562336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1023,0.05250666538874308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,31,0.9664959907531738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,2047,0.09386666615804036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,2047,0.05621333420276642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,4095,0.16945600509643555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,4095,0.09291733304659526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,8191,0.3213226596514384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,3,0.01834133391578992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1,0.0222080002228419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1,0.021669333179791767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,3,0.02214933435122172
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,3,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,7,0.021589333812395733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,7,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,15,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,31,0.021909333765506744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1023,0.03377600014209747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,31,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,8191,0.16487466295560202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,63,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,127,0.02125866711139679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,255,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,255,0.021013334393501282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,511,0.03143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,15,0.022154666483402252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1023,0.04748799900213877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1023,0.042261332273483276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,63,0.021397332350413006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,2047,0.07963733375072479
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,2047,0.06904533505439758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,4095,0.1377226710319519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,511,0.0276053324341774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,3,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,7,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,31,0.01156266654531161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,511,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,511,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1023,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,2047,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,2047,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,4095,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,4095,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,127,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,8191,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,16383,0.02939733366171519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1023,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,16383,0.025429333249727886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,15,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,31,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,127,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,511,0.012063999970753988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1023,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1023,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,2047,0.01681600014368693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,2047,0.015397333850463232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,4095,0.021226666867733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,4095,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,8191,0.03379199902216593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,8191,0.029477333029111225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,16383,0.05598400036493937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,16383,0.04791999856630961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1,0.03329066683848699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1,0.031141333281993866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,4095,0.11770666639010112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,3,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,7,0.03332799921433131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,7,0.03145600110292435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,15,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,15,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,31,0.03196266790231069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,31,0.031141333281993866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,63,0.032431999842325844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,63,0.0312266672650973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,127,0.03160533308982849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,127,0.03138133386770884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,255,0.03443733354409536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,3,0.03301866600910822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,255,0.03143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,511,0.04980800052483877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,511,0.04507199923197428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1023,0.07702933251857758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1023,0.06849066913127899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,2047,0.13621333241462708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,2047,0.11761599779129028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,8191,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,15,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,31,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,31,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,63,0.012698666503032049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,63,0.012266666938861212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,127,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,255,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,255,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,511,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1023,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1023,0.016293333222468693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,2047,0.02348266790310542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,2047,0.02316266546646754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,4095,0.036576000352700554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,8191,0.05922133227189382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,8191,0.0518453319867452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,16383,0.10150933265686035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,16383,0.08869333068529765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1,0.05640000104904175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1,0.052000001072883606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,3,0.05611200133959452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,3,0.05226133267084757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,7,0.0561653325955073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,7,0.0518506666024526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,15,0.056015998125076294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,15,0.05223466455936432
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,4095,0.031301334500312805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,31,0.05314666529496511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,63,0.05553600192070007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,63,0.051813334226608276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,127,0.05619200070699056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,127,0.050848002235094704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,255,0.058389330903689064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,255,0.051728000243504844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,511,0.08714133501052856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,511,0.07675200204054515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1023,0.13990933696428934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1023,0.1218293309211731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,1,0.10181867082913716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,1,0.09106133381525676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,3,0.10328533252080281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,3,0.09117333094278972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,7,0.10332266489664714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,7,0.09114133318265279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,15,0.10338667035102844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,15,0.09115200241406758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,31,0.10333866874376933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,31,0.09186666210492452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,63,0.10354133447011311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,63,0.09118400017420451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,127,0.10148800412813823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,127,0.09322133660316467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,255,0.10154666503270467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,255,0.09342933694521587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,1,0.1936053236325582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,1,0.17529600858688354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,3,0.19449067115783691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,3,0.1763733426729838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,7,0.19341333707173666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,7,0.17645333210627237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,15,0.19352533419926962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,15,0.17507733901341757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,31,0.19384533166885376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,31,0.1749066710472107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,63,0.1934666633605957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,63,0.17638399203618368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,127,0.18933866421381632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,127,0.17518933614095053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,3,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,31,0.056218668818473816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,3,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,7,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,7,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,15,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,15,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,31,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,31,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,63,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,63,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,127,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,127,0.013424000392357508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,255,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,511,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,511,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1023,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1023,0.01883200059334437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,2047,0.03186133255561193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,2047,0.027530667682488758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,4095,0.050101334849993386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,4095,0.043925335009892784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,8191,0.08088533580303192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,8191,0.07116266588370006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,16383,0.1439520021279653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,16383,0.1241386632124583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,1,0.3749013344446818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,1,0.3391840060551961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,3,0.37547731399536133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,3,0.33902935187021893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,7,0.37594131628672284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,7,0.33894399801890057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,15,0.37562131881713867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,15,0.3402719895044963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,31,0.33881068229675293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,31,0.3761386473973592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,63,0.37536001205444336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,63,0.33905065059661865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,1,0.7353706359863281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,1,0.6627680063247681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,3,0.7364532947540283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,3,0.6638720035552979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,7,0.7351360321044922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,7,0.6627519925435384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,15,0.7359039783477783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1,0.016224000602960587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,3,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,31,0.7360320091247559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,3,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,7,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,15,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,15,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,31,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,31,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,15,0.6628640095392863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,63,0.016623999923467636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,63,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,127,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,255,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,255,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,7,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,511,0.019519999623298645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1023,0.03164266546567281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1023,0.027488000690937042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,2047,0.051455999414126076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,127,0.01544533297419548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,4095,0.08307733138402303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,4095,0.0724480003118515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,511,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,8191,0.14628266294797262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,8191,0.12441600362459819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,3,0.02537599951028824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,3,0.02189333240191142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,2047,0.044490665197372437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,7,0.025290665527184803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,7,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,15,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,15,0.021338666478792827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,31,0.024197332561016083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,31,0.021359999974568684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,63,0.02332799881696701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,63,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,127,0.024122667809327442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,127,0.021365332106749218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,255,0.02571200082699458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,255,0.021557333568731945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,511,0.04374399781227112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,511,0.03121600051720937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1023,0.07275733351707458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1023,0.04584533472855886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,2047,0.13549333810806274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,2047,0.07728533446788788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,4095,0.25277332464853924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,31,0.6626026630401611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1,0.025173333783944447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,15,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,255,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1023,0.015487999965747198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1023,0.013349333157142004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,2047,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,2047,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,4095,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,4095,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,8191,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,8191,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,16383,0.04614933331807455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,16383,0.028725333511829376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,511,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,4095,0.13432000080744425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1023,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,2047,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,4095,0.029552000264326733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,4095,0.022842665513356526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,8191,0.0531626691420873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,8191,0.0335413341720899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,255,0.011717333147923151
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,16383,0.090229332447052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1023,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1,0.03849600007136663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1,0.0336053321758906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,3,0.03775999943415324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,3,0.033589333295822144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,7,0.03802666564782461
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,7,0.03342933456103007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,15,0.03774933268626531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,15,0.03338133295377096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,31,0.03802666564782461
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,31,0.03380800038576126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,63,0.03861333429813385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,63,0.03327466547489166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,127,0.04152533411979675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,127,0.03345066557327906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,255,0.04371733466784159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,511,0.07387199997901917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,511,0.048351998130480446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1023,0.12802666425704956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1023,0.075013334552447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,2047,0.24381333589553833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,16383,0.05613866448402405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1,0.01239466667175293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,3,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,3,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,7,0.012346666306257248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,7,0.012015999605258306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,15,0.012351999680201212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,15,0.012304000556468964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,31,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,31,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,127,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,255,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,255,0.035173334181308746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,511,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1023,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1023,0.015658666690190632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,2047,0.03033600002527237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,2047,0.02349333216746648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,4095,0.05622399846712748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,4095,0.035349334279696144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,511,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,2047,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,8191,0.05817066629727682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,16383,0.1742453376452128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1,0.06865600248177846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1,0.054698665936787925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,3,0.06841066479682922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,3,0.055120001236597695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,2047,0.13193066914876303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,7,0.06820266445477803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,7,0.05575466652711233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,16383,0.09854933619499207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,15,0.06846400101979573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,15,0.05415999889373779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,31,0.0684746652841568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,31,0.05565333366394043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,63,0.06856533388296764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,63,0.055919999877611794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,127,0.07051200171311696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,127,0.05588266750176748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,255,0.07266666491826375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,255,0.05829333265622457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,511,0.13078400492668152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,511,0.08366933465003967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1023,0.23894933859507242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,1,0.12523200114568075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,1,0.1034399966398875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,8191,0.09500267108281453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,3,0.12382400035858154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,3,0.1030506690343221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,15,0.1241973340511322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1023,0.13395733634630838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,15,0.10352533062299092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,31,0.12398933370908101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,31,0.10354133447011311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,63,0.12386666735013326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,63,0.10379200180371602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,127,0.12597866853078207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,127,0.10132267077763875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,7,0.1239466667175293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,255,0.13033599654833475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,255,0.1032533347606659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,7,0.10286933183670044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,1,0.23642667134602866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,3,0.23646400372187296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,3,0.19329599539438883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,7,0.23600532611211142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,7,0.19318399826685587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,15,0.23643199602762857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,15,0.1935946742693583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,31,0.2361066738764445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,31,0.19350399573644003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,63,0.23632532358169556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,63,0.1934453248977661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,1,0.19325866301854452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,127,0.2387253244717916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,127,0.19107200702031454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,3,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,3,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,7,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,7,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,15,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,15,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,31,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,31,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,63,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,63,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,127,0.014762666076421738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,255,0.01381333296497663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,255,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,511,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,511,0.015386667102575302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1023,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1023,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,2047,0.05128000179926554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,2047,0.03230933348337809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,4095,0.07930666704972585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,4095,0.04994666576385498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,8191,0.14102400342623392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,16383,0.2669493357340495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,16383,0.14070933063824972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,1,0.4578666687011719
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,8191,0.08072533210118611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,3,0.4580373366673787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,3,0.3726026614507039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,7,0.4581973155339559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,15,0.45772798856099445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,1,0.3721706469853719
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,15,0.3720959822336833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,31,0.45773335297902423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,63,0.4577813148498535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,7,0.3762933413187663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,31,0.37221864859263104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,1,0.9028586546579996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,1,0.7306719621022543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,63,0.37323200702667236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,3,0.9022186597188314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,7,0.9026827017466227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,7,0.7298986911773682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,15,0.9021600087483724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,15,0.7301653226216634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,3,0.729914665222168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,3,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,7,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,31,0.7303626537322998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,7,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,15,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,15,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,31,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,31,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,63,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,63,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,127,0.01681600014368693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,127,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,255,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,255,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,511,0.02518933266401291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,511,0.021530665457248688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,31,0.9004800319671631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1023,0.029535998900731403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,2047,0.0795253316561381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,2047,0.05013866722583771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,4095,0.14506666858990988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,4095,0.08076266447703044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,8191,0.26899733146031696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,8191,0.14229866862297058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,3,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,3,0.019167999426523846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,7,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,7,0.019130667050679524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,15,0.019738666713237762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,15,0.018746666610240936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,31,0.01987733319401741
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,31,0.01985599969824155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,3,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,127,0.020810666183630627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,127,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,255,0.01953599974513054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,255,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,511,0.027295999228954315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,511,0.024234667420387268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1023,0.04093866546948751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,63,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1023,0.03446933378775915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,63,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,2047,0.0687253326177597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,2047,0.06015466650327047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,4095,0.1157919963200887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1023,0.04493333399295807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,31,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,127,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,4095,0.10091200470924377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,255,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,511,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,511,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1023,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,2047,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,4095,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,4095,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,8191,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,8191,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,16383,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,7,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,31,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,511,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,511,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1023,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,3,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,2047,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,4095,0.021397332350413006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,4095,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,8191,0.03305066625277201
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,8191,0.030789333085219067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,16383,0.05610666672388712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,16383,0.047839999198913574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1,0.027647999425729115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,2047,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,3,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,7,0.029071999092896778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,7,0.027461332579453785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,15,0.029578665892283123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,15,0.02756800005833308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,16383,0.025306666890780132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,31,0.02769600103298823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,3,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,63,0.02714666724205017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,127,0.029274667302767437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,127,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,255,0.02959466725587845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,255,0.028037334481875103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,511,0.038218667109807335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1023,0.06699733436107635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,63,0.02942933390537898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1023,0.05890133480230967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,2047,0.11552000045776367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,2047,0.09986666838328044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,31,0.029370665550231934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,15,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,31,0.011653333902359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,31,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,63,0.01191466674208641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,127,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,511,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1023,0.016623999923467636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1023,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,2047,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,4095,0.03364799916744232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,511,0.043951998154322304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,4095,0.029792000850041706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,8191,0.059706668059031166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,7,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,16383,0.10154133041699727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,16383,0.08710400263468425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1,0.047930667797724404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1,0.04420800010363261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,3,0.048010667165120445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,3,0.043935999274253845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,7,0.04788800080617269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,7,0.043951998154322304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,15,0.047983999053637184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,15,0.04409066836039225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,31,0.047877331574757896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,31,0.04417600234349569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,63,0.04784533381462097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,8191,0.047930667797724404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,127,0.04771733283996582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,127,0.044079999128977455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,255,0.04991999765237173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,255,0.044309332966804504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,511,0.07460266848405202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,511,0.06631466746330261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,2047,0.023711999257405598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1023,0.11953066786130269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1023,0.10366933544476827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,1,0.08709333340326945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,1,0.07874133189519246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,3,0.08695466319719951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,3,0.07850666840871175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,7,0.08705066641171773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,7,0.07860800127188365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,63,0.044010668992996216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,15,0.07888533174991608
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,31,0.08683733145395915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,31,0.07785599927107494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,63,0.0869653324286143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,63,0.07859200239181519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,127,0.08508267005284627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,127,0.07876800000667572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,255,0.08686932921409607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,255,0.07889600098133087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,1,0.1646666626135508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,1,0.14850667119026184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,3,0.16394666830698648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,3,0.14841600259145102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,7,0.16461867094039917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,7,0.1482186714808146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,15,0.16454399625460306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,15,0.14829333623250326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,31,0.1646399994691213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,31,0.1467519998550415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,63,0.16470932960510254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,63,0.14827733238538107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,127,0.160970667997996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,15,0.08695466319719951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,3,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,7,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,7,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,15,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,31,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,31,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,127,0.14854400356610617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,63,0.013514666507641474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,63,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,127,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,127,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,255,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,255,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,511,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,511,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1023,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1023,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,2047,0.028762665887673695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,2047,0.026309333741664886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,4095,0.048250665267308555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,4095,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,8191,0.08082666496435802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,8191,0.06967466572920482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,16383,0.1446453332901001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,3,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,1,0.31404266754786175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,1,0.2842026750246684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,3,0.31445332368214923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,3,0.2836640079816182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,7,0.3143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,7,0.2842666705449422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,16383,0.12193600336710612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,15,0.31432000796000165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,15,0.2840106685956319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,31,0.31496532758076984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,31,0.28385066986083984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,63,0.3144426743189494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,63,0.2834879954655965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,1,0.61626664797465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,1,0.5558880170186361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,3,0.6153599818547567
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,3,0.5559039910634359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,7,0.6158080101013184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,7,0.5559680064519247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,15,0.6156426668167114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,15,0.5556586583455404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,3,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,3,0.015637333194414776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,31,0.615669329961141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,31,0.5558079878489176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,7,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,15,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,15,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,31,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,31,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,63,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,127,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,127,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,255,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,511,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,511,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1023,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1023,0.023610666394233704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,2047,0.04403733213742574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,2047,0.03754133234421412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,4095,0.07162666817506154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,4095,0.06265600025653839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,8191,0.12392533818880717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,8191,0.107205331325531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1,0.021221332252025604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,3,0.022042666872342426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,3,0.019519999623298645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,255,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,7,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,15,0.02311466634273529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,15,0.01958400011062622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,31,0.022517333428064983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,31,0.01959466685851415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,63,0.021482666333516438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,63,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,127,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,127,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,255,0.02288000037272771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,255,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,511,0.0382080003619194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,511,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1023,0.06224533418814341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1023,0.04038933416207632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,2047,0.11357333262761433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,2047,0.06737066805362701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,4095,0.20957867304484049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,4095,0.11343999703725179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,7,0.015322666615247726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,7,0.02180800090233485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,15,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,511,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1023,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1023,0.013951999445756277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,2047,0.014607999473810196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,4095,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,4095,0.016197333733240765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,8191,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,8191,0.01964266722400983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,16383,0.04191466669241587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,16383,0.025920001169045765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,511,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1,0.011749333391586939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,3,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,7,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,15,0.011781333635250727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,31,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,31,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,63,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,255,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,511,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,511,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1023,0.01340266689658165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,2047,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,2047,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,4095,0.02514133354028066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,4095,0.021477334201335907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,8191,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,16383,0.0876693328221639
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,16383,0.05435200035572052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1,0.03363733241955439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1,0.02921066681543986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,3,0.03385066737731298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,3,0.02922666569550832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,7,0.03357866654793421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1023,0.013669333110253016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,15,0.03350933392842611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,15,0.029343999922275543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,31,0.03350933392842611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,31,0.02945599953333537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,63,0.03349333256483078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,63,0.029274667302767437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,127,0.0353973334034284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,127,0.028016000986099243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,255,0.037952000896135964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,255,0.028933333853880566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,511,0.06258666515350342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,511,0.042175998290379844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1023,0.1083679993947347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1023,0.06436266501744588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,7,0.029872000217437744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,2047,0.11181333661079407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,3,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,3,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,7,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,7,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,15,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,31,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,31,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,63,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,63,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,127,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,2047,0.20558400948842367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,127,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,255,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,511,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,511,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,15,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1023,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,2047,0.027477333943049114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,255,0.01246400053302447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,2047,0.02350933353106181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,8191,0.0505973349014918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,4095,0.05526400109132131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,4095,0.033039999504884086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,8191,0.09387200077374776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,8191,0.057909334699312844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,16383,0.17132800817489624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1,0.05756799876689911
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1,0.047877331574757896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,3,0.05816000203291575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1023,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,7,0.0583840012550354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,7,0.04827199876308441
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,16383,0.09764267007509868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,15,0.0582826683918635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,15,0.04850133260091146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,31,0.047968000173568726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,63,0.05830933153629303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,127,0.061290666460990906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,3,0.04808000226815542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,255,0.06250133117039998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,255,0.04965866605440775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,511,0.10983999570210774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,31,0.0580213318268458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,511,0.07256000240643819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,63,0.04799999793370565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1023,0.1149066686630249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,1,0.10564266641934712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,127,0.047456001242001854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,3,0.10545066992441814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,3,0.08686400453249614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,7,0.10609599947929382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,7,0.0869706670443217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,15,0.1066986620426178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,15,0.08683733145395915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,31,0.10697600245475769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,31,0.08664533495903015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,63,0.10746133327484131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,1,0.08731200297673543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,127,0.10716799894968669
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,127,0.08694400389989217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,255,0.11179199814796448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,255,0.08735466996828715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,1,0.19845332702000937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,1,0.16335999965667725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1023,0.20136533180872598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,3,0.1987733244895935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,3,0.16330666343371072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,7,0.19946134090423584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,7,0.1630826691786448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,15,0.1997119983037313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,15,0.16340266664822897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,31,0.19855999946594238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,63,0.19898132483164468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,63,0.16295466820398966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,127,0.2009013295173645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,63,0.08703466256459554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,3,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,3,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,7,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,31,0.16403733690579733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,15,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,15,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,31,0.013674666484196981
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,31,0.013653332988421122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,127,0.15947199861208597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,63,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,63,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,127,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,255,0.014752000570297241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,255,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,7,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,511,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1023,0.02144533395767212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1023,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,2047,0.029552000264326733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,127,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,4095,0.07665599882602692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,4095,0.04827733337879181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,8191,0.1385546624660492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,8191,0.07998933394749959
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,511,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,16383,0.2589919964472453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,2047,0.04582933088143667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,1,0.3125706712404887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,3,0.38419198989868164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,3,0.3124693234761556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,16383,0.14038399855295816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,7,0.3832213481267293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,7,0.31267199913660687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,1,0.3829066753387451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,15,0.38359999656677246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,15,0.31270400683085126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,31,0.38386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,31,0.31306666135787964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,63,0.38365332285563153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,63,0.31834665934244794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,1,0.7526079813639323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,3,0.7534026304880778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,3,0.6108746528625488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,7,0.7533653577168783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,7,0.609941323598226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,15,0.7528639634450277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,1,0.6099253495534261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,3,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,3,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,7,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,31,0.7521706422170004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,7,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,15,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,31,0.01682666689157486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,31,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,63,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,63,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,127,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,127,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,31,0.6111253499984741
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,255,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,255,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,511,0.021055998901526134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,511,0.019658666104078293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1023,0.03961600114901861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1023,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,2047,0.07019199927647908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,2047,0.04442666471004486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,4095,0.12197333574295044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,4095,0.07044800122578938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,15,0.6102186838785807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,8191,0.22588266928990683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,3,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,3,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,7,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,15,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,7,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,15,0.017664000391960144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,15,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,31,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,31,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,63,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,63,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,127,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,8191,0.12222400307655334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,255,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,255,0.017674667139848072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,511,0.023381332556406658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,511,0.021525333325068157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1023,0.03562666724125544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1023,0.029765332738558452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,2047,0.0580266664425532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,2047,0.051413332422574363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,4095,0.09506666660308838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,4095,0.0812853326400121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,8191,0.1669386625289917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,8191,0.14242133498191833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,127,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,511,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1023,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1023,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,2047,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,4095,0.013530666629473368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,4095,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,8191,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,8191,0.01684800038735072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,16383,0.021344001094500225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,16383,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,32767,0.03568533311287562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,32767,0.029391999046007793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,3,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,2047,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,31,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,127,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,511,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1023,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1023,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,2047,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,2047,0.014554666976133982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,4095,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,4095,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,8191,0.022618666291236877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,8191,0.021189334491888683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,16383,0.035989334185918175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,16383,0.029311999678611755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,32767,0.05750399827957153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,32767,0.04975466430187225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1,0.024661332368850708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,3,0.025546667476495106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,3,0.02497066557407379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,7,0.025514667232831318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,7,0.024746666351954143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,15,0.025631998976071674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,15,0.02510933329661687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,31,0.025370667378107708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,31,0.024677333732446034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,63,0.02515200028816859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,127,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,127,0.02495466669400533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,255,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,255,0.025173333783944447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,511,0.031680000325044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1023,0.05602666735649109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1023,0.049786667029062905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,2047,0.09713066617647807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,2047,0.08505066235860188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,4095,0.17032533884048462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,4095,0.14657066265741983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,3,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,3,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,15,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,15,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,31,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,255,0.011968000481526056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,511,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,511,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1023,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1023,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,2047,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,2047,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,4095,0.023077333966890972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,4095,0.021317332983016968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,511,0.037791999677817024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,8191,0.03749333322048187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,8191,0.031194667021433514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,16383,0.05825066566467285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,16383,0.05187733471393585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,32767,0.10076266527175903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,32767,0.08716266353925069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1,0.03775999943415324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,3,0.03955200066169103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,3,0.03761066744724909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,63,0.02610666553179423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,7,0.0377813329299291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,15,0.040549332896868386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,31,0.040474665661652885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,31,0.038191998998324074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1,0.03948266555865606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,63,0.03740799923737844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,127,0.042266666889190674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,7,0.03990400085846583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,127,0.037530665596326195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,255,0.041797334949175514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,15,0.037685332198937736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,511,0.06201066573460897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,511,0.056373332937558494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1023,0.0969493289788564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1023,0.0867146650950114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,63,0.04063999901215235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,2047,0.1483733355998993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,1,0.07083733379840851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,1,0.06423466900984447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,3,0.07083733379840851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,255,0.037392000357309975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,2047,0.17105066776275635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,7,0.0645546664794286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,15,0.07027199864387512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,15,0.06419733166694641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,31,0.07051733136177063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,31,0.06439466774463654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,63,0.070592001080513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,63,0.06437333424886067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,3,0.0647626668214798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,127,0.07233066856861115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,7,0.07070399820804596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,255,0.07233066856861115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,255,0.06629866858323415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,511,0.11134399970372517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,511,0.09790933132171631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,1,0.1323093374570211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,1,0.11957866946856181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,3,0.11939199765523274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,7,0.13396267096201578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,7,0.11980266372362773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,15,0.13401599725087485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,15,0.11985599994659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,31,0.13214932878812155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,31,0.11982400218645732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,3,0.13331733147303262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,127,0.06426133215427399
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,127,0.11988266309102376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,63,0.1318986713886261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,63,0.11994133392969768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,255,0.11973866820335388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,3,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,7,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,15,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,63,0.011535999675591787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,127,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,127,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,511,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1023,0.016837333639462788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1023,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,2047,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,2047,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,127,0.1304586629072825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,4095,0.030645333230495453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,8191,0.06011733412742615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,8191,0.05189866820971171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,16383,0.10131733616193135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,16383,0.08788266777992249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,255,0.13080533345540366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,32767,0.18524799744288126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,1,0.2544320027033488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,4095,0.039520000418027244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,1,0.23033599058787027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,3,0.25386667251586914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,3,0.23066665728886923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,7,0.25325334072113037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,32767,0.15846400459607443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,15,0.2550453344980876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,15,0.23015467325846353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,31,0.2548639973004659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,31,0.23064533869425455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,63,0.253493328889211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,63,0.23024000724156699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,127,0.24888000885645548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,127,0.22882133722305298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,1,0.4965706666310628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,1,0.4481866757074992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,3,0.49515199661254883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,7,0.23061333100001016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,3,0.44781867663065594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,7,0.49666134516398114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,7,0.4477706750233968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,15,0.4964053233464559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,15,0.4482133388519287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,31,0.4965813159942627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,31,0.4478293259938558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,3,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,3,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,7,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,7,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,15,0.013418667018413544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,15,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,63,0.44768532117207843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,31,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,63,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,63,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,127,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,127,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,255,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,255,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,511,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,511,0.015962666521469753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1023,0.021503999829292297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,63,0.4966346820195516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,31,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,2047,0.031658666829268135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,4095,0.06227200229962667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,4095,0.054378668467203774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,8191,0.10362133383750916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,8191,0.08905599514643352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1023,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,2047,0.04037333279848099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,16383,0.18677333990732828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,16383,0.1586133340994517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,3,0.01921066641807556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,3,0.017727999637524288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,7,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,7,0.01758933315674464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,15,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,31,0.017909333109855652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,63,0.019189332922299702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,63,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,127,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,127,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,255,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,255,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,511,0.033520000676314034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,511,0.022831998765468597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1023,0.05197866757710775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1023,0.03364266703526179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,2047,0.09400000174840291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,4095,0.1698933243751526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,31,0.019850666324297588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,4095,0.09358400106430054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,8191,0.320853332678477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,2047,0.056015998125076294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,15,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,31,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,255,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,511,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,511,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1023,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,2047,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,4095,0.015504000087579092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,8191,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,8191,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,16383,0.03414933383464813
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,16383,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,32767,0.05527999997138977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,32767,0.034874667723973594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,511,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1023,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,2047,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,2047,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,4095,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,4095,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,8191,0.03645866612593333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,8191,0.02142400046189626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,16383,0.05737066765626272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,16383,0.03544000039498011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,32767,0.0977280040582021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,32767,0.05580266813437144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1,0.027930667002995808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,8191,0.1662613352139791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,3,0.029477333029111225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,7,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,7,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,15,0.029818666477998097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,15,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,31,0.029711998999118805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,31,0.025701334079106648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,63,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,3,0.025306666890780132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,2047,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,127,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,255,0.03301866600910822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,255,0.025263999899228413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,511,0.0525439977645874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,511,0.03765333443880081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1023,0.054373333851496376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,63,0.0249439999461174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,127,0.028970666229724884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,4095,0.3186933398246765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,4095,0.16701332728068033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,3,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,2047,0.16904000441233316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,15,0.011877333124478659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,31,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,31,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,63,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,63,0.011551999797423681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1023,0.09040000041325887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,127,0.011887999872366587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,2047,0.09502399961153667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,255,0.011839999506870905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,255,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,511,0.013616000612576803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1023,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,2047,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,2047,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,4095,0.037503999968369804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,4095,0.023237332701683044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,8191,0.05825600028038025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,511,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,8191,0.0376800000667572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,16383,0.09942400455474854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,16383,0.05829866727193197
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,32767,0.18131732940673828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,127,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1,0.04010133445262909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,3,0.047925333182017006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,3,0.03987200061480204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,7,0.04811733464399973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,7,0.03985599925120672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,32767,0.0972213347752889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1,0.04794133206208547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,7,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,31,0.039994666973749794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,63,0.04770666857560476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,63,0.039887999494870506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,127,0.04984533290068308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,15,0.04770666857560476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,15,0.039818666875362396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,31,0.04771733283996582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,511,0.09219200412432353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,511,0.060533334811528526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1023,0.16517333189646402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1023,0.09506133198738098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,255,0.05397866666316986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,255,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,2047,0.1684053341547648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,1,0.0869760016600291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,1,0.07049599786599477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,3,0.08659733335177104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,3,0.07082133491834004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,7,0.0858133335908254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,7,0.07066666583220164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,127,0.03956266740957896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,15,0.0705386648575465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,31,0.08675733208656311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,2047,0.3177280028661092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,63,0.0869760016600291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,63,0.07055999835332234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,127,0.08813866972923279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,127,0.07083199918270111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,255,0.09221333265304565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,255,0.07235733171304067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,511,0.1693120002746582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,511,0.10763733585675557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,1,0.16084266702334085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,1,0.13286933302879333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,3,0.16083199779192606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,3,0.13218667109807333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,7,0.1606880029042562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,7,0.13265066345532736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,15,0.0860746701558431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,15,0.16089066863059998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,15,0.13214932878812155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,31,0.16075199842453003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,31,0.13329600294431052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,63,0.1607306698958079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,63,0.13316266735394797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,127,0.16290133198102316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,127,0.13009066383043924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,255,0.170250674088796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,31,0.07035199801127116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,255,0.13202133774757385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,3,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,3,0.01259200026591619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,7,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,15,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,15,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,31,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,31,0.01239466667175293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,63,0.012543999900420507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,127,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,255,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,7,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,511,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1023,0.018272000054518383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1023,0.016197333733240765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,63,0.013487999637921652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,2047,0.02420266717672348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,4095,0.05983466903368632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,255,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,8191,0.10068800052007039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,8191,0.05858666698137919
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,16383,0.18361065785090128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,16383,0.09935466448465984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,32767,0.3485066493352254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,4095,0.039359999199708305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,32767,0.18061333894729614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,1,0.309717337290446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,3,0.31037867069244385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,2047,0.037392000357309975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,3,0.2532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,7,0.31015467643737793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,7,0.25305066506067914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,15,0.3096800049146016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,15,0.25483200947443646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,31,0.31034133831659955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,31,0.2532266577084859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,63,0.3102346658706665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,63,0.2528480092684428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,127,0.3160693248112996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,127,0.24869332710901895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,1,0.2540266712506612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,1,0.6050719817479452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,1,0.4925493399302165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,3,0.6057279904683431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,3,0.4926240046819051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,7,0.605023980140686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,7,0.49225600560506183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,15,0.6052906513214111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,15,0.493066668510437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,31,0.6039946476618449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,31,0.49253865083058673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,63,0.6050613323847452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,63,0.4925493399302165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,3,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,7,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,15,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,15,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,31,0.015781333049138386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,31,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,63,0.015552000453074774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,63,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,127,0.014741333822409311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,127,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,255,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,255,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,511,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,511,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1023,0.0341386670867602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1023,0.021935999393463135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,2047,0.05985599756240845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,2047,0.03815466662247976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,4095,0.10126399993896484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,4095,0.059989333152770996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,8191,0.18415466944376627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,3,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,7,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1,0.01648533344268799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,3,0.016805333395799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,3,0.018288000176350277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,8191,0.10133333007494609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,7,0.016303999970356624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,7,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,15,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,16383,0.34906665484110516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,31,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,16383,0.1833440065383911
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,63,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,63,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,127,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,127,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,255,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,255,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,511,0.021397332350413006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,511,0.019440000255902607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1023,0.03053866575161616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1023,0.02606400102376938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,2047,0.0503359983364741
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,2047,0.04359999795754751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,4095,0.08182933429876964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,4095,0.07156800230344136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,31,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,3,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,255,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,511,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,511,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1023,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1023,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,8191,0.12384532888730367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,2047,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,4095,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,4095,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,8191,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,15,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,8191,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,16383,0.021301334102948506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,2047,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,32767,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,8191,0.14629866679509482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,7,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,32767,0.029578665892283123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,63,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,16383,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,255,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,511,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1023,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1023,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,2047,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,2047,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,4095,0.017509333789348602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,4095,0.015562667200962702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,8191,0.021359999974568684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,8191,0.01884799947341283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,16383,0.02996266633272171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,16383,0.025306666890780132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,32767,0.04444799820582072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,3,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,3,0.021573332448800404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,7,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,7,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,15,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,15,0.021375998854637146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,31,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,31,0.021541332205136616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,63,0.021194666624069214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,63,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,127,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,127,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,255,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,255,0.021055998901526134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,32767,0.03774400055408478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1023,0.047279998660087585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1023,0.04018666595220566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,2047,0.07925866544246674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,2047,0.07043733199437459
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,4095,0.13819199800491333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,511,0.031317333380381264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,4095,0.11757333079973857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,15,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,127,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,511,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1023,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,2047,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,2047,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,4095,0.02309866746266683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,4095,0.021503999829292297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,8191,0.03547733277082443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,8191,0.029461334149042766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,16383,0.050293331344922386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,32767,0.09939733147621155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,32767,0.08758399883906047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1,0.03151999910672506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,3,0.03340800106525421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,3,0.031727999448776245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,7,0.03375466664632162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,7,0.03127466638882955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,16383,0.058117335041364036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,15,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,15,0.03127466638882955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,31,0.033488000432650246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1,0.033258666594823204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,63,0.03368533402681351
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,63,0.03149333347876867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,127,0.03366400053103765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,127,0.030981334547201794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,255,0.03357866654793421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,255,0.030261332790056866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,511,0.0499946673711141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,511,0.043925335009892784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1023,0.07863999903202057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1023,0.0684799998998642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,2047,0.13642133275667825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,31,0.03159466634194056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,1,0.05598933498064677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1023,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,1,0.051728000243504844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,3,0.056277334690093994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,3,0.05005866785844167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,7,0.05605333546797434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,7,0.051183998584747314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,15,0.05513600011666616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,15,0.05178666611512502
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,31,0.05621333420276642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,31,0.05186666548252106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,63,0.056015998125076294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,63,0.05194133520126343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,127,0.05600533386071523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,127,0.05226666728655497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,255,0.05797333518664042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,255,0.052629331747690834
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,511,0.08736532926559448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,511,0.07644266883532207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,1,0.10322667161623637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,1,0.09131200114885966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,3,0.10321066776911418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,3,0.090938667456309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,7,0.10326400399208069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,7,0.09103999535242717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,15,0.10264000296592712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,15,0.09122133255004883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,31,0.10348266363143921
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,2047,0.11762133240699768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,31,0.09087999661763509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,63,0.10336533188819885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,63,0.09119466940561931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,127,0.10120532910029094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,127,0.09298666318257649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,3,0.012613333761692047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,3,0.012634667257467905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,7,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,7,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,15,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,15,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,31,0.012341332932313284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,31,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,255,0.012426666915416718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,255,0.1036959985891978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,255,0.09321600198745728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,511,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1023,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1023,0.01552533358335495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,2047,0.02350933353106181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,2047,0.021920000513394673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,4095,0.03731200098991394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,4095,0.03159466634194056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,8191,0.060415998101234436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,8191,0.05063466727733612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,16383,0.10218133529027303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,16383,0.08699199557304382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,511,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,32767,0.18557866414388022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,32767,0.15786666671435037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,1,0.1951786677042643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,1,0.17664533853530884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,3,0.19532267252604166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,3,0.17715734243392944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,7,0.1956640084584554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,7,0.17682133118311563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,15,0.1959786613782247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,31,0.19554134209950766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,31,0.17709867159525552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,63,0.19521600008010864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,63,0.1770346760749817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,127,0.19133333365122476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,127,0.17601599295934042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,15,0.17544533809026083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,1,0.3766080141067505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,1,0.33904000123341876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,3,0.3756106694539388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,3,0.33902935187021893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,7,0.3762773275375366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,7,0.33894399801890057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,15,0.37590400377909344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,15,0.33905065059661865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,31,0.3393813371658325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,63,0.3758399883906047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,3,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,3,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,7,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,63,0.33909865220387775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,7,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,15,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,15,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,31,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,31,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,63,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,127,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,255,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,255,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,511,0.01471466695268949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1023,0.02006400004029274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,2047,0.032773333291212715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,2047,0.027797333896160126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,4095,0.05101333558559418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,31,0.37620266278584796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,8191,0.08227733274300893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,511,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,8191,0.07046933472156525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1023,0.01886933296918869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,16383,0.1439466675122579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,16383,0.1232373317082723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,3,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,3,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,7,0.01739199956258138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,7,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,15,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,15,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,31,0.017738666385412216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,31,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,63,0.017701332767804463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,63,0.01716800034046173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,127,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,127,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,255,0.018709332992633183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,255,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,511,0.025786665578683216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,511,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1023,0.044794668753941856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1023,0.02938666691382726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,2047,0.08083199958006541
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,2047,0.049728001157442726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,4095,0.14288533727327982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,4095,0.04257600009441376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,8191,0.26980799436569214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,8191,0.1427786648273468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,4095,0.08117866516113281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,2047,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,4095,0.015642666568358738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,8191,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,8191,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,16383,0.025936000049114227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,16383,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,32767,0.045754666129748024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,32767,0.0276053324341774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,7,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,4095,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,63,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,127,0.011781333635250727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,255,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,255,0.01181866725285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1023,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1023,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,2047,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,4095,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,4095,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,8191,0.02645866572856903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,511,0.0122079998254776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,8191,0.020949333906173706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,16383,0.04593066871166229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,16383,0.028666667640209198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,32767,0.07504533231258392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,32767,0.0447626660267512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,2047,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,3,0.021525333325068157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,7,0.02548266698916753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,7,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,15,0.023520000278949738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,15,0.021375998854637146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,31,0.02329600105683009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,31,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,63,0.02348800003528595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,63,0.021615999440352123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,127,0.023530667026837666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,127,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,255,0.026506667335828144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,255,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,511,0.04433600107828776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,511,0.031258667508761086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1023,0.07272000114123027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1023,0.04456533491611481
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,2047,0.1360426644484202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,4095,0.2529439926147461
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,4095,0.13503467043240866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,3,0.02458133300145467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,7,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,2047,0.07743466893831889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,63,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,255,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,511,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1023,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1023,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,2047,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,2047,0.018496000518401463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,4095,0.03199466566244761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,4095,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,8191,0.054469332098960876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,8191,0.035530666510264076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,16383,0.09153067072232564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,16383,0.05728533367315928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1,0.02458133300145467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,32767,0.09724799791971843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1,0.03790933390458425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1,0.033402666449546814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,3,0.03894400099913279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,3,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,7,0.03956266740957896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,7,0.03369600077470144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,15,0.03823466598987579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,15,0.03334933271010717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,31,0.03377600014209747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,63,0.03942399968703588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,63,0.03328000009059906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,32767,0.16749866803487143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,127,0.03230399886767069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,255,0.04408533374468485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,255,0.03475199888149897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,511,0.07155199845631917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,511,0.050186668833096824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1023,0.12752532958984375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1023,0.07634666562080383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,31,0.03822933385769526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,127,0.04156800111134847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,1,0.05630933245023092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,3,0.06772266825040181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,3,0.05646933118502299
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,7,0.06837333242098491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,7,0.05619200070699056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,1,0.06858666737874348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,15,0.06834666430950165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,15,0.053914666175842285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,31,0.068271999557813
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,31,0.05442666510740916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,63,0.06771733363469441
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,127,0.06876266499360402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,2047,0.24476265907287598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,127,0.055498664577802025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,255,0.07234666744867961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,255,0.058058664202690125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,511,0.12956800063451132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,511,0.08488532900810242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,1,0.12396799524625142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,63,0.05429333448410034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,3,0.12385066350301106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,2047,0.13408533732096353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,7,0.12369599938392639
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,7,0.10380267103513081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,15,0.12416533629099528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,15,0.10385599732398987
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,31,0.1253493328889211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,31,0.10361066460609436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,63,0.12505066394805908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,3,0.10326400399208069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,127,0.1260479986667633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,127,0.10347200433413188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,255,0.1325759987036387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,3,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,3,0.01211200033624967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,63,0.10342400272687276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,7,0.011605333536863327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,7,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,15,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,31,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,31,0.012666666259368261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,255,0.10532800356547038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,63,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,63,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,127,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,127,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,255,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,511,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,511,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1023,0.018458666900793713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1023,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,15,0.012538666526476542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,2047,0.02351466566324234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,4095,0.056261335810025535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,4095,0.03543466577927271
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,8191,0.09428800145785014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,16383,0.17313599586486816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,1,0.10378133257230122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,16383,0.09890133142471313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,32767,0.3306506673494975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,32767,0.17996267477671304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,8191,0.05845333139101664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,1,0.23507734139760336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,1,0.19344000021616617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,3,0.23449599742889404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,2047,0.031290667752424874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,3,0.19336533546447754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,7,0.23455466826756796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,7,0.19362133741378784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,15,0.23439999421437582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,15,0.1934559941291809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,31,0.23561600844065347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,31,0.19448000192642212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,63,0.2347360054651896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,63,0.19477866093317667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,127,0.23690134286880493
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,127,0.19169066349665323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,1,0.45794133345286053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,3,0.4573333263397217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,1,0.3742560148239136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,7,0.4577440023422241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,7,0.37373332182566327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,15,0.45771201451619464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,3,0.37336532274882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,15,0.3737119833628337
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,31,0.4575466712315877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,31,0.37387200196584064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,3,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,63,0.4573333263397217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,63,0.37437868118286133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,7,0.013877333452304205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,15,0.014485333114862442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,15,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,31,0.01463466634353002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,31,0.013679999858140945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,63,0.015413332730531693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,63,0.014618666221698126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,127,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,7,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,255,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,255,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,511,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,511,0.015509333461523056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1,0.013631999492645264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1023,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,2047,0.048154667019844055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,3,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,2047,0.03137599925200144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,127,0.014554666976133982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,4095,0.049216002225875854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,8191,0.14241600036621094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,8191,0.07996266583601634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,16383,0.26740266879399616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1023,0.025536000728607178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,3,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,3,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,4095,0.0794293334086736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,7,0.013349333157142004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,15,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,15,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,16383,0.14236266414324442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,63,0.013701333353916803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,63,0.014015999933083853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,127,0.01379199946920077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,7,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,255,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,255,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,31,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,511,0.016757333030303318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,31,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1023,0.021456000705560047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,2047,0.039701332648595176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,2047,0.0316746657093366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,4095,0.062394668658574425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,4095,0.05400000015894572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,127,0.013450667262077332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,8191,0.10342933734258015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,8191,0.08901866277058919
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,16383,0.15929067134857178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1023,0.023157333334287006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,16383,0.18761066595713297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,127,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,255,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,511,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,2047,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,2047,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,4095,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,4095,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,8191,0.015471999843915304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,8191,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,16383,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,16383,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,32767,0.021130666136741638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,65535,0.03550933301448822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,32767,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,65535,0.029557332396507263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,15,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,31,0.010309333602587381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,127,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,127,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,255,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,511,0.012106666962305704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,511,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,2047,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,2047,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,4095,0.014389333625634512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,8191,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,8191,0.016506666938463848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,16383,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,32767,0.03575466573238373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,32767,0.029701332251230877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,65535,0.05598400036493937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,65535,0.05012266834576925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,3,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,3,0.017717332889636356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,7,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,7,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,15,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,15,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,31,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,31,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,63,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,63,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,127,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,127,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,255,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,255,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,511,0.023007998863856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,511,0.021359999974568684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1023,0.03543466577927271
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1023,0.029669334491093952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,2047,0.05799466868241628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,2047,0.051818668842315674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,4095,0.09518933296203613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,4095,0.08268799881140391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,8191,0.16883732875188193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,4095,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,3,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,7,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,2047,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,2047,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,4095,0.017818666994571686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,4095,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,8191,0.023178666830062866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,8191,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,16383,0.03749866783618927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,16383,0.03146666785081228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,32767,0.05832533538341522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,32767,0.052383999029795326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,65535,0.10153067111968994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,65535,0.08727467060089111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,8191,0.14385599891344705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1,0.02446399877468745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,3,0.023887999355793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,7,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,7,0.023984000086784363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,15,0.02514133354028066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,15,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,31,0.025173333783944447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,31,0.02348800003528595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,63,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,63,0.023552000522613525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,3,0.025114665428797405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,127,0.025061334172884624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,127,0.02347733328739802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,255,0.025242666403452556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,255,0.02384000023206075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,511,0.03875733415285746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,511,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1023,0.05644799768924713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1023,0.049925332268079124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,2047,0.09739200274149577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,2047,0.08508800466855367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,4095,0.1712906757990519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1,0.037530665596326195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,3,0.03775999943415324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,7,0.04037333279848099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,7,0.037952000896135964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,15,0.0401653324564298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,15,0.03756266583998998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,31,0.039781334499518074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,31,0.037962667644023895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,63,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,63,0.03741333385308584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,127,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,127,0.03786666691303253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,3,0.04018666595220566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,255,0.03742400060097376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,511,0.06227200229962667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,511,0.054746667544047035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1023,0.09779199957847595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1023,0.08483733733495076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,1,0.07086400190989177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,1,0.06435733536879222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,4095,0.14601066708564758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,3,0.07074666519959767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,255,0.042026668787002563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,7,0.07035733262697856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,7,0.06431999802589417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,15,0.07091199855009715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,15,0.06433066725730896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,31,0.07055999835332234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,31,0.06467199822266896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,63,0.07038400073846181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,63,0.06439466774463654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,127,0.07142400244871776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,127,0.06412800153096516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,255,0.0726453314224879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,255,0.06673066814740498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,511,0.11018133163452148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,3,0.06401066482067108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,255,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1023,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,2047,0.017530667285124462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,511,0.09716266393661499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,4095,0.02145066608985265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,8191,0.03787733366092046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,8191,0.031290667752424874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,16383,0.05853333572546641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,16383,0.05201066533724467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,32767,0.10077866911888123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,32767,0.08710400263468425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,2047,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,4095,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,65535,0.18387732903162637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,65535,0.15804266929626465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,1,0.13430399696032205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,1,0.11966400345166524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,3,0.1341546674569448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,3,0.11959466338157654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,7,0.13409599661827087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,7,0.11989333232243855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,15,0.1344533363978068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,15,0.11956800023714702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,31,0.1341600020726522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,31,0.11965866883595784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,63,0.1322879989941915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,63,0.11954666177431743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,127,0.13057066996892294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,127,0.11966933806737264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,255,0.1321386694908142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,255,0.12054933110872905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,1,0.2553973396619161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,1,0.2302773396174113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,3,0.2548373341560364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,7,0.25490667422612506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,3,0.23069866498311362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,7,0.23052799701690674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,15,0.255018671353658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,15,0.23021332422892252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,31,0.25514666239420575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,31,0.2305013338724772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,63,0.25462400913238525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,63,0.23041067520777384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,127,0.230512003103892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,7,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,15,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,15,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,31,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,31,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,63,0.013418667018413544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,63,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,127,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,127,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,255,0.01173866664369901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,255,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,511,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,511,0.013445333888133367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1023,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1023,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,127,0.24946133295694986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,4095,0.039994666973749794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,4095,0.031583999594052635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,8191,0.060191998879114784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,8191,0.052517334620157875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,16383,0.10196266571680705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,16383,0.0888159970442454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,2047,0.024469333390394848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,32767,0.18508267402648926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,2047,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,3,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,3,0.014757333944241205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,7,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,7,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,15,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,15,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,31,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,31,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,63,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,63,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,127,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,127,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,255,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,255,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,511,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,32767,0.15868799885114035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1023,0.03356799980004629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1023,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,2047,0.05996799965699514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,2047,0.03953066716591517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,4095,0.10178132851918538
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,4095,0.06118399898211161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,8191,0.18293333053588867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,8191,0.10234133402506511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,16383,0.3485333522160848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,511,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,7,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,15,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,31,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,16383,0.18343466520309448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,63,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,255,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,7,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,511,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1023,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,2047,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,2047,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,4095,0.015589332828919092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,4095,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,8191,0.01877333347996076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,8191,0.01624533285697301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,16383,0.02311466634273529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,16383,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,32767,0.03733866661787033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,32767,0.023498666783173878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,65535,0.0358240008354187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,3,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,7,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,511,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,2047,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,65535,0.05602133274078369
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,127,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,4095,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,4095,0.015498666713635126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,8191,0.019989332805077236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,8191,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,16383,0.03575466573238373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,16383,0.022805333137512207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,15,0.011621333658695221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,32767,0.055999999245007835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,32767,0.03619199991226196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,65535,0.09692266583442688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1,0.01887999971707662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,65535,0.05613866448402405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,3,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,7,0.01951466624935468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,7,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,15,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,15,0.01923199991385142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,31,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,31,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,63,0.02089066555102666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,63,0.017594666530688603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,127,0.019797333826621372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,127,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,255,0.01998399943113327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,255,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,2047,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,511,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1023,0.05195199946562449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1023,0.034517332911491394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,2047,0.0939520001411438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,2047,0.05676266551017761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,4095,0.16902933518091837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,4095,0.09346133470535278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,8191,0.3203786611557007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,511,0.03365866591533025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,3,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,31,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,127,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,511,0.011829332758982977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,511,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,8191,0.16788800557454428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,2047,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,4095,0.02149333308140437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,4095,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,8191,0.03957866628964742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1023,0.013493333011865616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,2047,0.017914666483799618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,16383,0.058543999989827476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,16383,0.037733333806196846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,32767,0.09900800387064616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,32767,0.058634668588638306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,65535,0.1807253360748291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,8191,0.023584000766277313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,3,0.02917333443959554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,3,0.025120000044504803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,7,0.029738667110602062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,7,0.02513066679239273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,15,0.028202667832374573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,15,0.025962665677070618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,31,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,63,0.029232000311215717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,63,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,127,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,127,0.02536533276240031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,255,0.03313066562016805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,65535,0.09805867075920105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,255,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,511,0.03781333317359289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1023,0.09123733639717102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1023,0.05561600128809611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,31,0.0283146674434344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,2047,0.16943466663360596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,511,0.053898667295773826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,4095,0.32019732395807904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1,0.03967999915281931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,3,0.04780266682306925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,3,0.03995199998219808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,7,0.047925333182017006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,7,0.04001600046952566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,4095,0.1688906749089559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,15,0.04721599817276001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,15,0.03998400022586187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,31,0.047541335225105286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,31,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,63,0.047728002071380615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,63,0.04057066639264425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,127,0.05014933149019877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,127,0.039834665755430855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,255,0.05287466446558634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,255,0.04270400106906891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1,0.048197334011395775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,511,0.06036800146102905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1023,0.16456533471743265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1023,0.09527466694513957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,1,0.08680533369382222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,1,0.07084799806276958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,2047,0.09515200058619182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,3,0.08680533369382222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,3,0.07055999835332234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,7,0.08561600248018901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,7,0.07084799806276958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,15,0.08717333277066548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,15,0.07031466563542683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,31,0.08690133690834045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,31,0.07065600156784058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,63,0.08637866377830505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,63,0.07045333087444305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,127,0.08716799815495808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,127,0.07041066884994507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,255,0.09198932846387227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,255,0.07293866574764252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1,0.01259200026591619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,511,0.1693440079689026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,3,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,7,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,511,0.09249599774678548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,511,0.10778666536013286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,3,0.011653333902359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,255,0.012245333443085352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,511,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,511,0.011866666376590729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1023,0.014346666634082794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1023,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,2047,0.019487999379634857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,2047,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,127,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,4095,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,8191,0.059157331784566246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,8191,0.03836799909671148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,16383,0.09798399607340495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,16383,0.0581226646900177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,4095,0.03771200031042099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,32767,0.1797013282775879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,32767,0.09774399797121684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,65535,0.34298133850097656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,65535,0.17798932393391928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,1,0.16129600008328757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,1,0.1339893341064453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,3,0.16058666507403055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,3,0.1339040001233419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,7,0.1609493295351664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,7,0.13201600313186646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,15,0.16153599818547568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,15,0.1338933308919271
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,31,0.16085333625475565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,31,0.13431466619173685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,63,0.1612213353315989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,63,0.13262400031089783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,127,0.1629759967327118
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,127,0.13133333126703897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,255,0.1713013251622518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,1,0.31030933062235516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,1,0.25481067101160687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,3,0.3102773427963257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,3,0.2550560037295024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,7,0.3100693424542745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,7,0.2547946572303772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,15,0.3099466760953267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,15,0.2548533280690511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,255,0.13296533624331155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,31,0.25494933128356934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,63,0.3096746603647868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,63,0.25446399052937824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,127,0.3123253385225932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,31,0.31028799215952557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,3,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,3,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,7,0.013418667018413544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,7,0.012058666596810022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,15,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,15,0.01249066616098086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,31,0.012426666915416718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,63,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,63,0.012517333030700684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,127,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,255,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,255,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1,0.012256000190973282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,511,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1023,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1023,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,2047,0.037845333417256675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,2047,0.024149333437283833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,127,0.011823999385039011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,4095,0.06015466650327047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,4095,0.039349332451820374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,8191,0.10142933328946431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,511,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,127,0.24900799989700317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,16383,0.18337599436442056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,16383,0.09954667091369629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,32767,0.34759998321533203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,3,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,3,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,7,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,15,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,8191,0.06029333174228668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,31,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,31,0.013354666531085968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,63,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,63,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,32767,0.18133334318796793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,127,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,255,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,255,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,15,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,511,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1023,0.019845332950353622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1023,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,2047,0.03382933388153712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,2047,0.027461332579453785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,4095,0.05091199775536855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,4095,0.04318933188915253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,8191,0.08276266853014629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,8191,0.07072000205516815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,127,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,16383,0.14547200004259744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,511,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,7,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,127,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,255,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,511,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,511,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,2047,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,4095,0.015402667224407196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,4095,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,8191,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,8191,0.015429332852363586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,16383,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,16383,0.12371200323104858
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,16383,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,2047,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,32767,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,32767,0.019754666835069656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,3,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,65535,0.02972800036271413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,127,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1023,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,2047,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,4095,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,4095,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,8191,0.017664000391960144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,65535,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,16383,0.02125866711139679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,16383,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,32767,0.03133333226044973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,32767,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,65535,0.04601066807905833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,65535,0.04138133426507314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,3,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,3,0.01685333376129468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,7,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,7,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,15,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,15,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,31,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,31,0.01533866673707962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,63,0.015685333559910457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,63,0.015381333728631338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,127,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,255,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,255,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,511,0.021018666525681812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,511,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1023,0.029637334247430164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,8191,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,2047,0.05170666674772898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,2047,0.04387733340263367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,4095,0.08270933230717976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,4095,0.07070399820804596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,8191,0.14560533563296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,8191,0.12397332986195882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,7,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,31,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,63,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,127,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1023,0.014698666830857595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1023,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,2047,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,2047,0.01587733378012975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,4095,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,255,0.010527999450763067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,8191,0.021551998953024547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,8191,0.01939733326435089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,16383,0.03127466638882955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,16383,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,32767,0.04684799909591675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1023,0.0276053324341774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,32767,0.03949866692225138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,65535,0.07611200213432312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1,0.02237333357334137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,3,0.022842665513356526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,4095,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,7,0.021770666042963665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,15,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,15,0.021269333859284718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,31,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,65535,0.06644266843795776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,31,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,63,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,127,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,3,0.02142400046189626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,7,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,255,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,511,0.031983998914559685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,511,0.027424000203609467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1023,0.04790399968624115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1023,0.04116799930731455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,127,0.020997333029905956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,2047,0.07901866734027863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,2047,0.07021333277225494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,4095,0.13693333665529886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,4095,0.1181653340657552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1,0.033626665671666466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1,0.03180799881617228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,3,0.03345066557327906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,3,0.03137599925200144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,7,0.03316800047953924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,7,0.031311998764673867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,15,0.033520000676314034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,255,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,31,0.033530667424201965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,31,0.03134933362404505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,63,0.03323733309904734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,63,0.031311998764673867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,127,0.03323200096686681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,127,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,63,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,255,0.031744000812371574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,511,0.0499946673711141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,511,0.04403733213742574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,15,0.031178665657838184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1023,0.07760533193747203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1023,0.06868800024191539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,1,0.0553653339544932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,1,0.05179733534653982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,3,0.055829331278800964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,3,0.05037866532802582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,7,0.055919999877611794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,7,0.05199466645717621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,255,0.03477333237727483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,15,0.05618133147557577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,15,0.04996799925963084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,31,0.055861334005991616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,31,0.05175999800364176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,63,0.05607999861240387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,63,0.05013333261013031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,127,0.056608001391092934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,127,0.050144001841545105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,255,0.057989334066708885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,255,0.052144000927607216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,31,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,511,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,511,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1023,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1023,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,2047,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,2047,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,4095,0.02316266546646754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,4095,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,8191,0.035546667873859406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,8191,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,16383,0.05819733440876007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,16383,0.05106666684150696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,32767,0.09924266735712688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,511,0.087226668993632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,511,0.0767626663049062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,32767,0.08693333466847737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,65535,0.18313600619633993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,65535,0.15662399927775064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,1,0.10261866450309753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,1,0.09149332841237386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,3,0.10340799887975057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,7,0.1032480001449585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,7,0.09129599730173747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,15,0.10320533315340678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,15,0.09117333094278972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,31,0.10363733768463135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,31,0.09134933352470398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,63,0.10308800141016643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,63,0.09086400270462036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,127,0.10132267077763875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,255,0.10315733154614766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,255,0.09355200330416362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,3,0.09097599983215332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,1,0.1954453388849894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,1,0.17725332578023276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,3,0.19530133406321207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,3,0.17693867286046347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,7,0.19593600432078043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,7,0.17638933658599854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,127,0.09283733367919922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,15,0.19545066356658936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,15,0.17699732383092245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,31,0.1950719952583313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,31,0.17709332704544067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,63,0.19553599754969278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,127,0.19158399105072021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,127,0.17710399627685547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,7,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,15,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,31,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,63,0.17705066998799643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,127,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,127,0.012074666718641916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,255,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,511,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,511,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,15,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1023,0.016810666769742966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1023,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,2047,0.023733332753181458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,2047,0.02250666668017705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,4095,0.03779733429352442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,4095,0.030671998858451843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,8191,0.0605973352988561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,255,0.01166933278242747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,16383,0.08716799815495808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,32767,0.18542933464050293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1,0.013749333719412485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,32767,0.15822933117548624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,3,0.01368533323208491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,3,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,7,0.013754667093356451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,7,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,15,0.013487999637921652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,31,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,31,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,63,0.014736000448465347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,63,0.014357333381970724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,127,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,127,0.013429333766301474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,255,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,255,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,511,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,511,0.015765332927306492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1023,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1023,0.019546666493018467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,2047,0.050383999943733215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,2047,0.03366933266321818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,4095,0.0806933343410492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,4095,0.05005866785844167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,8191,0.14241600036621094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,8191,0.08109333117802937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,16383,0.2671893239021301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,15,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,16383,0.14219733079274496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,16383,0.1014400025208791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,63,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,8191,0.05022933085759481
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,511,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,2047,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,2047,0.012282667060693106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,4095,0.016879999389251072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,4095,0.01534933348496755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,8191,0.01762666677435239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,8191,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,16383,0.01912533367673556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,32767,0.026975999275843304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,32767,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,65535,0.04668800036112467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,65535,0.02951466788848241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,16383,0.01966933285196622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,63,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1023,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,2047,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,2047,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,4095,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,8191,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,8191,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,16383,0.02554133286078771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,16383,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,32767,0.0480320006608963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,32767,0.029951999584833782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,4095,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,65535,0.04604266583919525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,3,0.01889066646496455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,3,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,7,0.018144000321626663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,7,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,15,0.01814933369557063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,15,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,31,0.018645333747069042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,31,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,63,0.018485333770513535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,63,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,127,0.017808000246683758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,127,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,255,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,255,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,511,0.025583999852339428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,511,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1023,0.0441599984963735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1023,0.029578665892283123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,2047,0.08055999875068665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,2047,0.051130667328834534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,4095,0.1426400045553843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,4095,0.08261866867542267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,8191,0.2694399952888489
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,8191,0.14383999506632486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,3,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,15,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,127,0.012469333906968435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,511,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,511,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1023,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,2047,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,2047,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,4095,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,4095,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,8191,0.027509334186712902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,65535,0.07654400169849396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,8191,0.023247999449570973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,16383,0.0488373339176178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,32767,0.07698133091131847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,32767,0.04690133531888326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,65535,0.1346453328927358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1,0.023610666394233704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,3,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,3,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,7,0.023317334552605946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,7,0.02292266736427943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,15,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,15,0.023061332603295643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,31,0.02348800003528595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,31,0.021295999487241108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,63,0.02516266703605652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,16383,0.0336053321758906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,127,0.021520001192887623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,255,0.026538667579491932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,255,0.021354667842388153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,511,0.043525333205858864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,511,0.031770666440327965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1023,0.07258666555086772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1023,0.04634666442871094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,2047,0.13487999637921652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,127,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,65535,0.07433600227038066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,4095,0.13594667116800943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1,0.03827200084924698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1,0.03356266766786575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,3,0.03797333439191183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,2047,0.07886933286984761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,7,0.03405333310365677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,4095,0.252949337164561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,15,0.03849600007136663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,15,0.03339733431736628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,31,0.038373333712418876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,31,0.033413333197434746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,63,0.03807466725508372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,63,0.03352533280849457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,3,0.033359999457995095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,127,0.03164266546567281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,255,0.042837331692377724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,255,0.03512533257404963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,511,0.07262933254241943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,511,0.049685334165891014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1023,0.12731732924779257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1023,0.07493333518505096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,1,0.06833066542943318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,1,0.05593066910902659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,127,0.04138133426507314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,3,0.06854933500289917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,3,0.05414933462937673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,7,0.06782400111357371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,7,0.05625066657861074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,15,0.06632533172766368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,15,0.055434669057528176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,31,0.06866133213043213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,31,0.05416533350944519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,63,0.06761066615581512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,63,0.055904000997543335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,127,0.06858133276303609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,127,0.05635733405749003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,255,0.07267733414967854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,255,0.05773866673310598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,511,0.13032000263532004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,511,0.08471999565760295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,7,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,7,0.03905066599448522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,63,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,511,0.011754666765530905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,511,0.012266666938861212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1023,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1023,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,2047,0.019679999599854153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,2047,0.018085333208243053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,4095,0.03161599983771642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,8191,0.05468800167242686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,8191,0.035461333890755974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,16383,0.09261866410573323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,32767,0.1677280068397522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,4095,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,32767,0.09718933701515198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,16383,0.05671466886997223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,65535,0.31675199667612713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,65535,0.1770133376121521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,1,0.12593600153923035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,1,0.10364266236623128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,3,0.1258133351802826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,3,0.10356799761454265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,7,0.12577066818873087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,7,0.10417067011197408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,15,0.1035146713256836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,31,0.12568533420562744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,31,0.10414399703343709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,63,0.12487467130025227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,63,0.10381866494814555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,127,0.12607466181119284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,127,0.1032319962978363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,255,0.13179733355840048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,255,0.10635733604431152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,15,0.12481066584587097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,1,0.23601067066192627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,1,0.1934773325920105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,3,0.23653332392374674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,3,0.19338667392730713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,7,0.2365973393122355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,7,0.1937333345413208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,15,0.23492799202601114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,15,0.19403199354807535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,31,0.23473066091537476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,31,0.19350934028625488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,63,0.23603200912475586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,63,0.19357333580652872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,3,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,7,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,15,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,127,0.19070400794347128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,63,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,127,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,255,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,511,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,511,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,127,0.23751999934514365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,31,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1023,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,2047,0.03166933357715607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,2047,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,4095,0.0565226674079895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,4095,0.03597866743803024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,8191,0.09563733140627544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,16383,0.17336533466974893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1023,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,32767,0.329365332921346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,3,0.011839999506870905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,7,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,15,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,15,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,31,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,32767,0.179365336894989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,127,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,127,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,255,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,511,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1023,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1023,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,2047,0.024090667565663654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,2047,0.022885332504908245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,16383,0.09917333722114563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,4095,0.03995733211437861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,4095,0.03166399896144867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,8191,0.06069866816202799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,8191,0.053157334526379905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,16383,0.10315733154614766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,8191,0.059893334905306496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,16383,0.08871466914812724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,32767,0.18605866034825644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,32767,0.15838932991027832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,127,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1023,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,4095,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,4095,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,8191,0.01569066693385442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,8191,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,16383,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,16383,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,32767,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,32767,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,2047,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,65535,0.023311999936898548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,31,0.012213333199421564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,131071,0.03850133220354716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,131071,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,65535,0.023391999304294586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,127,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,511,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,4095,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,7,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,8191,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,8191,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,16383,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,16383,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,32767,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,32767,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,4095,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,65535,0.03752533346414566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,65535,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,3,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,3,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,7,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,7,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,15,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,131071,0.057071998715400696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,31,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,131071,0.05074666440486908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,31,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,63,0.014688000082969666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,127,0.01444799949725469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,255,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,255,0.014101333916187286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,511,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,511,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1023,0.023034666975339253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,15,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1023,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,63,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,127,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,4095,0.0613919993241628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,4095,0.054117331902186074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,8191,0.1035093367099762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,8191,0.0890933374563853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,16383,0.18874132633209229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,16383,0.15986133615175882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,2047,0.04091199984153112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,2047,0.03133866687615713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,511,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,511,0.01156266654531161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,4095,0.015754666179418564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,8191,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,8191,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,16383,0.023039999107519787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,16383,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,32767,0.029685333371162415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,65535,0.0580266664425532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,65535,0.0517493337392807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,4095,0.01626666635274887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1,0.01759999990463257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,3,0.017840000490347546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,32767,0.039850667119026184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,3,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,7,0.017477333545684814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,7,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,15,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,15,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,31,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,31,0.01747200017174085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,63,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,63,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,127,0.017429333180189133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,127,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,255,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,255,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,511,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,511,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1023,0.034101332227389015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1023,0.029472000896930695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,2047,0.05811200042565664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,2047,0.051872000098228455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,4095,0.09476799766222636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,4095,0.08294400076071422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,131071,0.10318400462468465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,8191,0.14411733547846475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,131071,0.08874133229255676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,3,0.025733334322770435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,3,0.025301332275072735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,7,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,7,0.02565866708755493
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,15,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,15,0.025349333882331848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,31,0.025562666356563568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,31,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,8191,0.16883200407028198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,63,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,127,0.023557332654794056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,255,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,255,0.02348266790310542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,511,0.03741333385308584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,511,0.03175999969244003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1023,0.05602666735649109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1023,0.05013333261013031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,63,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,2047,0.08556266625722249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,127,0.025253333151340485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1,0.037690666814645134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,3,0.04035199930270513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,3,0.037717332442601524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,7,0.03979199876387914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,7,0.03770666569471359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,15,0.040218666195869446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,15,0.03796799977620443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,31,0.04090133309364319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,31,0.037471999724706016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,2047,0.09717866778373718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,63,0.04015466570854187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,63,0.03770133356253306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,127,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,127,0.03770133356253306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,255,0.041797334949175514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,255,0.03736000011364619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,511,0.062362665931383766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,511,0.05457599957784017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1023,0.09729066491127014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1023,0.08522666494051616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,15,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,15,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,511,0.012128000458081564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1023,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,2047,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,2047,0.015461333096027374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,4095,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,4095,0.01886933296918869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,8191,0.023765332996845245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,8191,0.02327466756105423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,16383,0.04022933294375738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,16383,0.031317333380381264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,32767,0.06039466460545858
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,32767,0.052426666021347046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,65535,0.10133866469065349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,65535,0.08707732955614726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,1,0.0711359977722168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,131071,0.18714133898417154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,1,0.0644160012404124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,3,0.07074666519959767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,7,0.07148799796899159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,7,0.0641599992911021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,15,0.07048533360163371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,15,0.06411733229955037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,131071,0.15820800264676413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,3,0.06428800026575725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,31,0.06460799773534139
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,63,0.07084799806276958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,63,0.06411733229955037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,127,0.07221333185831706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,127,0.06411733229955037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,255,0.07207466661930084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,255,0.06646400193373363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,31,0.07116266588370006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,511,0.11166399717330933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,511,0.09678933024406433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,1,0.13411200046539307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,1,0.11934933066368103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,3,0.13412266969680786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,3,0.11784000198046367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,7,0.13402666648228964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,7,0.1195146640141805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,15,0.13402666648228964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,15,0.1188320020834605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,31,0.1342026690642039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,31,0.11966400345166524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,63,0.1337546706199646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,63,0.11955733100573222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,127,0.13192533453305563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,255,0.1327946682771047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,15,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,63,0.010506667196750641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,255,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,511,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1023,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1023,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,255,0.12011733651161194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,7,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,4095,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,4095,0.021695998807748158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,8191,0.03931200007597605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,8191,0.02994133283694585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,16383,0.057962665955225624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,16383,0.0517439991235733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,127,0.12001599868138631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,32767,0.1011946698029836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,32767,0.08679466446240743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,65535,0.18318400780359903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,65535,0.15779200196266174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,7,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,7,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,2047,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,15,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,31,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,3,0.013605333864688873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,63,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,63,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,127,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,255,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,511,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1023,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,2047,0.03941333293914795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,2047,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,4095,0.061573331554730736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,4095,0.04008000095685323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,8191,0.10301867127418518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,8191,0.059818665186564125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1023,0.016821333517630894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,16383,0.18487467368443808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,16383,0.10167466600735982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,32767,0.3511039813359578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,32767,0.1813760002454122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,3,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,31,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,15,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,511,0.011866666376590729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,2047,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,2047,0.012149333953857422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,4095,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,4095,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,8191,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,8191,0.015568000574906668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,31,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,16383,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,32767,0.023168000082174938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,32767,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,31,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,65535,0.03887466589609782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,65535,0.02350933353106181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,16383,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,131071,0.05877333382765452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,3,0.011936000237862269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,131071,0.04043733328580856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,2047,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,2047,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,4095,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,8191,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,511,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,16383,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,16383,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,32767,0.03786666691303253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,32767,0.023845332364241283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,4095,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,65535,0.057664001981417336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,65535,0.037503999968369804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,3,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,7,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,8191,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,15,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,15,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,31,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,31,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,63,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,63,0.013690666606028875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,127,0.015360000232855478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,127,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,255,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,255,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,131071,0.057962665955225624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,511,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1023,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,2047,0.05977066854635874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,2047,0.03946666667858759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,4095,0.10133866469065349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,511,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,8191,0.18504534165064493
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,8191,0.10139200091362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,131071,0.09784000118573506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,16383,0.352725346883138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1023,0.03577066709597906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,15,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,4095,0.05994133154551188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,511,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,2047,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,2047,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,4095,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,8191,0.021013334393501282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,8191,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,16383,0.038831998904546104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,16383,0.024282666544119518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,32767,0.0592853327592214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,32767,0.03849600007136663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,16383,0.18441067139307657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,65535,0.058133333921432495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1,0.019146667172511418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,3,0.019839999576409657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,3,0.018250666558742523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,7,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,7,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,15,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,15,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,31,0.01939733326435089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,31,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,63,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,63,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,127,0.019493332753578823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,127,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,255,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,255,0.02081599955757459
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,511,0.03341866781314214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,131071,0.1853440006573995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,65535,0.09959466258684795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1023,0.05312533179918925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1023,0.034261333445707955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,2047,0.09524800380071004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,2047,0.05797333518664042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,4095,0.17086933056513467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,4095,0.09398933251698811
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,131071,0.09970133503278096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,8191,0.32542399565378827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,8191,0.16643733779589334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1,0.02937600016593933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,3,0.02752000093460083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,7,0.029130667448043823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,7,0.025242666403452556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,15,0.029103999336560566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,15,0.025050667424996693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,31,0.02977599948644638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,31,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,511,0.02313599983851115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,63,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,127,0.030896000564098358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,127,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,255,0.033386667569478355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,255,0.025125332176685333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,511,0.0524479995171229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,511,0.037861332297325134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,63,0.028954667349656422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1023,0.05535466472307841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,3,0.025616000096003216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,2047,0.17230933904647827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,2047,0.09533333778381348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1,0.0480320006608963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1,0.039850667119026184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1023,0.09119466940561931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,3,0.0480320006608963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,3,0.04002666721741358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,7,0.04800533254941305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,7,0.0400693342089653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,15,0.04808533191680908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,15,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,31,0.0480373352766037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,31,0.03966933240493139
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,63,0.0476800004641215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,127,0.0400693342089653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,255,0.052890668312708534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,255,0.042917331059773765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,511,0.09127466877301534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,511,0.060405333836873375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1023,0.1653333306312561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,63,0.03974399964014689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,3,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1023,0.09506666660308838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,63,0.01163200040658315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,63,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,127,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,511,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,511,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,2047,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,2047,0.01716800034046173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,4095,0.021301334102948506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,4095,0.019440000255902607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,8191,0.0397173340121905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,8191,0.02513066679239273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,16383,0.06002133091290792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,16383,0.03946666667858759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,32767,0.10007466872533162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,32767,0.058186665177345276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,65535,0.18381865819295248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,65535,0.09732799728711446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,127,0.050437331199645996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,1,0.08595732847849528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,1,0.07082666456699371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,3,0.08674666285514832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,131071,0.35174401601155597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,3,0.07051733136177063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,7,0.08585066596666972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,7,0.07084799806276958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,15,0.0867146650950114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,15,0.07031466563542683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,31,0.08665600419044495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,31,0.07075733443101247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,63,0.08584533135096233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,63,0.07050133248170216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,127,0.08690133690834045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,127,0.07047999898592631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,255,0.09174933036168416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,255,0.07229333122571309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,511,0.16725865999857584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,131071,0.18053332964579263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,1,0.16076800227165222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,3,0.16099199652671814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,3,0.1320853332678477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,511,0.10779199997584026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,7,0.16076800227165222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,7,0.1341600020726522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,15,0.1606613298257192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,15,0.1338986655076345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,31,0.1609280010064443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,31,0.1337386667728424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,63,0.16102932890256247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,63,0.1340000033378601
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,127,0.16268266240755716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,127,0.13197867075602213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,255,0.1707680026690165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,255,0.1344319979349772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,15,0.012080000092585882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,3,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,511,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,1,0.13447466492652893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1023,0.014757333944241205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1023,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,2047,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,4095,0.03818666686614355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,4095,0.02460266649723053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,8191,0.05973333120346069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,8191,0.04005333284536997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,16383,0.099781334400177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,16383,0.05870933334032694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,2047,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,32767,0.09883733590443929
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,65535,0.34828801949818927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,15,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,65535,0.17858133713404337
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,32767,0.18231467405954996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,127,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,511,0.011760000139474869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,511,0.012005332857370377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1023,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,2047,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,2047,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,4095,0.023050665855407715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,4095,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,8191,0.03951466580231985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,8191,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,16383,0.05982933441797892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,16383,0.052144000927607216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,32767,0.10051199793815613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,32767,0.08719999591509502
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1,0.009658666948477427
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,3,0.010490667074918747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,63,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,65535,0.1831573247909546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,255,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,65535,0.1586720049381256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1023,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,2047,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,4095,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,4095,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,8191,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,8191,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,511,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,16383,0.01941866676012675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,16383,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,2047,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,32767,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,65535,0.022831998765468597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,65535,0.02149333308140437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,131071,0.028543998797734577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,131071,0.027119999130566914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,15,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,32767,0.020015999674797058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,31,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,63,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,63,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,2047,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,4095,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,4095,0.01575999955336253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,8191,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,8191,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,16383,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,16383,0.017818666994571686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,2047,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,32767,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,32767,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,65535,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,65535,0.023077333966890972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1,0.011722666521867117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,131071,0.03987200061480204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,3,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,131071,0.031445334355036415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,31,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,63,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,63,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,127,0.012666666259368261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,7,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,127,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,511,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,511,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1023,0.016879999389251072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1023,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,2047,0.024112001061439514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,2047,0.023007998863856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,4095,0.039520000418027244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,4095,0.031301334500312805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1023,0.011658667276302973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,8191,0.06065600117047628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,8191,0.05264000097910563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,255,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,16383,0.10363200306892395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,16383,0.08892800410588582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,3,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,15,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,63,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,32767,0.18683199087778726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,32767,0.15877866744995117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,511,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,2047,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,2047,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,4095,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,4095,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,8191,0.01766933376590411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,8191,0.01754133279124896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,16383,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,16383,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,32767,0.02476799984773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,32767,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,65535,0.039701332648595176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,65535,0.03162666658560435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,3,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,131071,0.053397332628568016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,7,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,15,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,15,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,31,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,31,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,131071,0.060421332716941833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,63,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,7,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,127,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,127,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,255,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,255,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,511,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,511,0.016869333883126576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,63,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1023,0.021488000949223835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1023,0.021007999777793884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,2047,0.040463998913764954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,2047,0.031317333380381264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,4095,0.06237866481145223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,4095,0.05388799806435903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,3,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,8191,0.10488532980283101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,8191,0.0890826682249705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,3,0.017802666872739792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,16383,0.15876799821853638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,7,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,7,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,15,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,31,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,31,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,16383,0.1890773375829061
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,63,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,3,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,63,0.01770666614174843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,127,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,15,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,255,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,511,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,511,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1023,0.035445332527160645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1023,0.02921066681543986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,2047,0.058389330903689064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,2047,0.050000001986821495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,255,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,4095,0.09611200292905171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,4095,0.08091199894746144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,3,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,3,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,7,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,7,0.023381332556406658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,15,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,15,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,31,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,31,0.02515733242034912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,63,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,63,0.023391999304294586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,127,0.02521066615978877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,127,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,255,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,255,0.023423999547958374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,511,0.037402667105197906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,511,0.03166399896144867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,127,0.017509333789348602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1023,0.049770668148994446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,2047,0.09918399651845296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,31,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,127,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,255,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,511,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1023,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1023,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,2047,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,4095,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,4095,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,8191,0.01912533367673556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,8191,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,2047,0.08494933446248372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,16383,0.021456000705560047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,32767,0.0396373321612676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1023,0.05614933371543884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,65535,0.06018666426340739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,16383,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,65535,0.05176533261934916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,32767,0.030975999931494396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1,0.03999999910593033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,131071,0.10321066776911418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,131071,0.08881066242853801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,3,0.039690665900707245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,3,0.03759466608365377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,7,0.03742400060097376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1,0.03749333322048187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,15,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,15,0.03751466671625773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,7,0.04015466570854187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,31,0.039477333426475525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,31,0.03739733248949051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,63,0.03997866561015447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,63,0.03748800108830134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,127,0.03976000100374222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,127,0.037733333806196846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,255,0.04154133299986521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,255,0.036789332826932274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,511,0.06205333272616068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,511,0.05494933327039083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1023,0.09724799791971843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1023,0.08455999692281087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,1,0.07081600030263265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,3,0.07063999772071838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,3,0.06426133215427399
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,7,0.07055999835332234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,7,0.0642080008983612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,15,0.07050133248170216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,1,0.06446933249632518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,15,0.06452799836794536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,31,0.07065066695213318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,31,0.06463466584682465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,63,0.07073600093523662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,63,0.06432533264160156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,127,0.07074666519959767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,127,0.06413333117961884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,255,0.07253333429495494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,255,0.06658133367697398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,511,0.09725866715113322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,15,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,127,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,127,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,255,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1023,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,2047,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,511,0.11134933431943257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,2047,0.015664000064134598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,4095,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,8191,0.02359466751416524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,8191,0.022917332748572033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,16383,0.03975466638803482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,16383,0.031498665610949196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,32767,0.05994133154551188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,32767,0.05272533496220907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,65535,0.10100799798965454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,65535,0.08760533730189006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,3,0.011674666156371435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,7,0.012367999802033106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,131071,0.1874826749165853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,131071,0.15842666228612265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,63,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,511,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1023,0.014101333916187286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1023,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,2047,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,4095,0.039434666434923805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,4095,0.023370665808518726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,8191,0.059903999169667564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,8191,0.03871466716130575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,16383,0.10078932841618855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,16383,0.058101331194241844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,32767,0.18121600151062012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,2047,0.01863466699918111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,65535,0.3453173240025838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,32767,0.09910933176676433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,7,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,7,0.012714666624863943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,63,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,65535,0.17914666732152304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,127,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,511,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,15,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,511,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1023,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,2047,0.012730666746695837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,4095,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,4095,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,8191,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,8191,0.015626666446526844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,16383,0.018895999838908512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,16383,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,32767,0.021503999829292297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,32767,0.019621333728233974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1023,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,65535,0.024890666206677754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,65535,0.0215786670645078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,15,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,31,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,31,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,131071,0.043477331598599754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,255,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,131071,0.027535999814669292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,2047,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,2047,0.012272000312805176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,4095,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,4095,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,8191,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,16383,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1023,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,16383,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,32767,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,32767,0.02089600016673406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,8191,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,65535,0.039887999494870506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,65535,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,3,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,3,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,7,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,15,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,131071,0.04065066576004028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,31,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,31,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,63,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,63,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,127,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,127,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,255,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,255,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,511,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,511,0.01393066719174385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1023,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1023,0.01646399994691213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,2047,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,131071,0.06031466523806254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,2047,0.02312533309062322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,4095,0.05974400043487549
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,4095,0.03968533376852671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,8191,0.10129066308339436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,8191,0.06028800209363302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,16383,0.18329066038131714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,7,0.013455999394257864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,32767,0.3471466700236003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,32767,0.18352532386779785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,127,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,255,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,255,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,511,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1023,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,2047,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,2047,0.015872000406185787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,4095,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,4095,0.017765333255132038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,8191,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,8191,0.019541333119074505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,16383,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,16383,0.021759999295075733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,32767,0.041375999649365745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,32767,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,65535,0.059978668888409935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,65535,0.03994133323431015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,16383,0.1011893351872762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,3,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,7,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,131071,0.06029333174228668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,15,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,15,0.013584000368913015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,31,0.013536000003417334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,63,0.015360000232855478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,63,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,131071,0.10042666395505269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,127,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,7,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,255,0.015439999600251516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,255,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,511,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,511,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1023,0.035386666655540466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1023,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,3,0.015461333096027374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,2047,0.06032533446947733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,2047,0.040074666341145836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,4095,0.10229866703351338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,8191,0.1843306620915731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,8191,0.10334933797518413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,16383,0.35024531682332355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,16383,0.1852746605873108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,4095,0.06243733565012614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,3,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,3,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,7,0.019823999454577763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,7,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,15,0.018960000326236088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,15,0.0189280000825723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,31,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,31,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,63,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,63,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,127,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,127,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,255,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,255,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,511,0.03363200028737386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,511,0.023056000471115112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1023,0.05314666529496511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1023,0.035690667728583016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,2047,0.09638399879137675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,2047,0.058090666929880776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1,0.01939733326435089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,127,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,4095,0.09525866309801738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1,0.025146665672461193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,3,0.029306667546431225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,3,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,7,0.02938666691382726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,7,0.025087999800841015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,15,0.02924266705910365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,15,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,31,0.029146666328112285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,31,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,63,0.029498666524887085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,63,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,4095,0.17116800944010416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,127,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,255,0.033733333150545754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,255,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,511,0.05188799897829691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,511,0.037658666570981346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1023,0.09074133634567261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1023,0.055013333757718406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,2047,0.1711946725845337
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,2047,0.09707732995351155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,3,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,7,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,31,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,511,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1023,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1023,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,2047,0.015498666713635126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,63,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,4095,0.017818666994571686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,8191,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,8191,0.021946666141351063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,127,0.029264000554879505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,16383,0.039674667020638786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,16383,0.023434666295846302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,2047,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,32767,0.038133333126703896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,4095,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,65535,0.10060800115267436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,65535,0.05981333553791046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,32767,0.05832533538341522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,131071,0.18416533867518106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,131071,0.10011200110117595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1,0.039818666875362396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,3,0.04769066472848257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,3,0.040021332601706185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,7,0.039808000127474465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1,0.048165331284205117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,15,0.04771733283996582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,15,0.03955733279387156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,31,0.047824000318845115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,63,0.04772266745567322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,7,0.04780800143877665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,127,0.049770668148994446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,127,0.0397119993964831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,255,0.052426666021347046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,255,0.04154133299986521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,63,0.039647998909155525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,511,0.09070932865142822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,511,0.06010666489601135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1023,0.16547733545303345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1023,0.09514133135477702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,1,0.08676266670227051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,1,0.07070399820804596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,3,0.08706133564313252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,3,0.07030933101971944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,31,0.040048000713189445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,7,0.08685866991678874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,7,0.07044800122578938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,15,0.08681600292523702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,15,0.07037866612275441
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,31,0.08717333277066548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,31,0.07051200171311696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,63,0.08715732892354329
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,63,0.0706879993279775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,127,0.07035199801127116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,255,0.0925439993540446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,255,0.0726560006539027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,511,0.16516266266504923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,3,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,7,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,511,0.10765332976977031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,511,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,127,0.08714133501052856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,2047,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,2047,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1023,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,4095,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,8191,0.0397119993964831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,8191,0.0232640008131663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,16383,0.05942399799823761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1023,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,16383,0.037834666669368744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,4095,0.021503999829292297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,32767,0.09943999846776326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,65535,0.1816106637318929
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,65535,0.09713066617647807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,131071,0.3466026782989502
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,131071,0.17844800154368082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,63,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,32767,0.05817066629727682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,511,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,511,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,31,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,2047,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,2047,0.01782400036851565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,4095,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,4095,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,8191,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1023,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,8191,0.023007998863856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,16383,0.039450667798519135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,16383,0.03142933299144109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,32767,0.0602400004863739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,32767,0.05346666773160299
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,15,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,65535,0.10113599896430969
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,65535,0.08694932858149211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,3,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,7,0.010512000570694605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,7,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,15,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,31,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,131071,0.15889599919319153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,63,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,2047,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,4095,0.011685332904259363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,8191,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,8191,0.015957333147525787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,16383,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,131071,0.18796799580256143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,16383,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,2047,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,4095,0.011685332904259363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,65535,0.023552000522613525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,65535,0.025306666890780132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,32767,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,131071,0.027488000690937042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,131071,0.026501332720120747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,7,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,15,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,31,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,127,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,255,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,255,0.0102613332370917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,32767,0.022944000860055287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1023,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1,0.010144000252087912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,2047,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,4095,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,4095,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,8191,0.016869333883126576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,8191,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,16383,0.01759999990463257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,16383,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,32767,0.020997333029905956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,32767,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,65535,0.02143999934196472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,65535,0.021301334102948506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,131071,0.029264000554879505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,3,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,131071,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,511,0.012671999633312225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,511,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,63,0.011637333780527115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1023,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,2047,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,2047,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,4095,0.02313599983851115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,4095,0.021525333325068157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,8191,0.039162665605545044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,8191,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,16383,0.05992533266544342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,16383,0.052943999568621315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,32767,0.10229866703351338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1023,0.013455999394257864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,3,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,3,0.012223999947309494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,7,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,15,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,31,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,31,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,63,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,65535,0.18417600790659586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,255,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,32767,0.08880000313123067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,65535,0.15869866808255514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,511,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,2047,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,2047,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,4095,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,4095,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,8191,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,8191,0.017605333278576534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,16383,0.01913600042462349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,16383,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,32767,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,32767,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,65535,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,65535,0.024485332270463307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,255,0.010512000570694605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,131071,0.04194133480389913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,3,0.012608000387748083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,3,0.012165332833925882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,131071,0.03332266708215078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,7,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,15,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,31,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,63,0.012448000411192576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,7,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,255,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,511,0.014469332993030548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1023,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1023,0.016549333930015564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,2047,0.024506665766239166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,2047,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,4095,0.04055999964475632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,4095,0.031498665610949196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,15,0.011957333733638128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,8191,0.06216000020503998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,8191,0.05389333268006643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,16383,0.10454400380452473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,16383,0.08902933200200398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,32767,0.1874720056851705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,3,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,3,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,32767,0.15916799505551657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,7,0.013440000514189402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,15,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,15,0.013770667215188345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,31,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,63,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,63,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,127,0.014287999520699183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,127,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,7,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,511,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,31,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1023,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1023,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,255,0.014720000326633453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,2047,0.04072533299525579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,2047,0.03173333406448364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,511,0.01684800038735072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,4095,0.0626986672480901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,4095,0.054373333851496376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,255,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,8191,0.08899199962615967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,3,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,3,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,7,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,7,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,15,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,15,0.01685333376129468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,31,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,31,0.016783999900023144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,63,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,8191,0.10533333818117778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,127,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,127,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,255,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,255,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,511,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,511,0.0210506667693456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1023,0.035391998787721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1023,0.028815999627113342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,2047,0.05970133344332377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,2047,0.05190933247407278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,7,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,15,0.010522666076819101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,31,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,63,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,4095,0.0971999963124593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,127,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,4095,0.0827893316745758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1023,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,2047,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,63,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,4095,0.015509333461523056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,4095,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,8191,0.017727999637524288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,8191,0.017973333597183228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,63,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,16383,0.020693333198626835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,32767,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,32767,0.024127999941507976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,65535,0.039861333866914116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,65535,0.031514666974544525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,16383,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1,0.025605333348115284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,131071,0.0617439995209376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,131071,0.05377600093682607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1,0.023344000180562336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,3,0.02514133354028066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,3,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,2047,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,7,0.023621333142121632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,15,0.02521066615978877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,15,0.02349333216746648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,7,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,31,0.023728000621000927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,63,0.025263999899228413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,63,0.025066666305065155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,127,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,31,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,255,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,255,0.0235359991590182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,511,0.037903999288876854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,127,0.025146665672461193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1023,0.05606933434804281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1023,0.04996799925963084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,2047,0.09963732957839966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,511,0.031845333675543465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,2047,0.08478400111198425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1,0.03961600114901861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1,0.0378506655494372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,3,0.04019733270009359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,3,0.037658666570981346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,7,0.03989866624275843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,7,0.03766933331886927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,15,0.04022400081157684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,15,0.03738133360942205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,31,0.03975466638803482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,31,0.03721600025892258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,63,0.040218666195869446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,63,0.03793599953254064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,127,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,127,0.03772266705830892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,255,0.043920000394185386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,255,0.03774400055408478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,511,0.06252799928188324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,511,0.054485330979029335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,7,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1023,0.08418132861455281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1023,0.09781333804130554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,7,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,255,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,511,0.012309333930412928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1023,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1023,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,2047,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,2047,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,4095,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,4095,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,127,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,8191,0.01947733387351036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,8191,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,255,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,16383,0.022944000860055287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,32767,0.03875733415285746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,32767,0.029781334102153778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,65535,0.06043733159701029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,65535,0.05180799961090088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,16383,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,131071,0.1037600040435791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,15,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,31,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,131071,0.08921066919962566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,511,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1023,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,2047,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,2047,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,4095,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,4095,0.01939733326435089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,8191,0.040234667559464775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,8191,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,16383,0.06117866436640421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,16383,0.03982933362325033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,31,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,32767,0.10095999638239543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,32767,0.05827199916044871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,65535,0.18258132537206015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,65535,0.09949333469072978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,3,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,131071,0.3466240167617798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,131071,0.18075732390085855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,127,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,255,0.011898666620254517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,511,0.01228800043463707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,511,0.01251199965675672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1023,0.011695999652147293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,2047,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,63,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,4095,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,8191,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,8191,0.01672533278663953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,16383,0.020351999749739964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1023,0.01192533348997434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,2047,0.01202133297920227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,32767,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,255,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,65535,0.02611200014750163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,65535,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,16383,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,127,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,131071,0.02941333254178365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,131071,0.028165332973003387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1023,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1023,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,2047,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,2047,0.012719999998807907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,32767,0.023050665855407715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,4095,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,8191,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,8191,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,16383,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,16383,0.019226666539907455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,32767,0.023386667172114056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,32767,0.020986666282018025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,4095,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,65535,0.025605333348115284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,65535,0.022965334355831146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,3,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,131071,0.045237332582473755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,131071,0.029616000751654308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,31,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,63,0.012634667257467905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,255,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,255,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,511,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1023,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,2047,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,4095,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,4095,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,8191,0.060415998101234436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,8191,0.039850667119026184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,7,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,16383,0.10105599959691365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,16383,0.05799466868241628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,2047,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,32767,0.18225600322087607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,32767,0.09924266735712688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,7,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,65535,0.17967466513315836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,127,0.011733333269755045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,65535,0.34481600920359295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,511,0.011594666788975397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,511,0.011578666667143503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1023,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,2047,0.015925332903862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,2047,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,4095,0.016879999389251072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,4095,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,8191,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,8191,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,16383,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,16383,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,32767,0.023717333873112995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,32767,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,65535,0.04171733558177948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,255,0.011850666254758835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1,0.013936000565687815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1,0.01368533323208491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,131071,0.06228800117969513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,3,0.013493333011865616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,131071,0.04398933549722036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,7,0.01368533323208491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,7,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,15,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,15,0.012304000556468964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,31,0.012650666137536367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,63,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,63,0.011973333855470022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,127,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,127,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,255,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,255,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,511,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,511,0.013653332988421122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,65535,0.025173333783944447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1023,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,2047,0.039546666045983635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,2047,0.023168000082174938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,4095,0.06037333110968272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,4095,0.040181333820025124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,8191,0.10134399930636089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,8191,0.06112533311049143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,3,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1023,0.020687999824682873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,16383,0.18363734086354574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1,0.013674666484196981
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,3,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,32767,0.18264534076054892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,3,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,7,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,16383,0.10123200217882793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,15,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,15,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,31,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,63,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,63,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,127,0.014741333822409311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,255,0.01544533297419548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,255,0.013834666460752487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,32767,0.3496319850285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,511,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1023,0.03584533433119456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,31,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1023,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,511,0.01887999971707662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,4095,0.10317333539326985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,4095,0.06252266466617584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,8191,0.18572266896565756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,8191,0.10545066992441814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,2047,0.041434665520985924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1,0.019146667172511418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1,0.017845333864291508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,3,0.020181333025296528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,7,0.01532799998919169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,3,0.018330667167901993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,7,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,7,0.01887999971707662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,15,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,15,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,31,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,31,0.018063999712467194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,63,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,63,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,127,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,127,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,255,0.019386666516462963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,255,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,511,0.033520000676314034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,511,0.023061332603295643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1023,0.05351999898751577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1023,0.033957332372665405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,2047,0.09723732868830363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,2047,0.059077332417170204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,4095,0.0965333382288615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,31,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,63,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,127,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,7,0.012453333785136541
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1023,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1023,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,2047,0.015354666858911514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,2047,0.015333333363135656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,4095,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,255,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,4095,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,8191,0.01951466624935468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,8191,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,16383,0.02362666775782903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,16383,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,32767,0.04182399809360504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,65535,0.06012799839178721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,65535,0.04009599983692169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,4095,0.1732800006866455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,131071,0.10053867101669312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,131071,0.06058666606744131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,32767,0.02568000058333079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,3,0.029274667302767437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,3,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1,0.027866666515668232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,7,0.029178666571776073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,7,0.02515200028816859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,15,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,15,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,31,0.029343999922275543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,31,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,63,0.02938666691382726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,63,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,127,0.029343999922275543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,127,0.025381334125995636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,255,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,511,0.0544106662273407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,511,0.03806933263937632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1023,0.09250133236249287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1023,0.05603733162085215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,255,0.03346133232116699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,2047,0.1748159925142924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,2047,0.09814400474230449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1,0.0484746644894282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1,0.039664000272750854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,3,0.047653332352638245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,3,0.040106666584809623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,7,0.04806933303674062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,7,0.03975466638803482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,15,0.04801600178082784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,15,0.040021332601706185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,31,0.04828799764315287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,31,0.03978666663169861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,63,0.0476693312327067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,63,0.04010133445262909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,127,0.04990399877230326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,127,0.03942399968703588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,255,0.0525546669960022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,255,0.04188799858093262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,511,0.09143466750780742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,511,0.06033066908518473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,3,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,7,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1023,0.16730666160583496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,15,0.01682666689157486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1023,0.09489599863688152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,15,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,511,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,2047,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,2047,0.015498666713635126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,4095,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,4095,0.01590399940808614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,8191,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,16383,0.039962666730086006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,16383,0.023978665471076965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,32767,0.06058666606744131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,32767,0.03885866701602936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,65535,0.10127466917037964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,8191,0.019509332875410717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,131071,0.18477332592010498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,7,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,131071,0.1011306643486023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,255,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,255,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,511,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,511,0.011909333368142446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,2047,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,2047,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,4095,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,4095,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,8191,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,8191,0.018160000443458557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,16383,0.023621333142121632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,16383,0.022618666291236877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1023,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,32767,0.039605334401130676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,32767,0.03153600047032038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,65535,0.06010666489601135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,65535,0.05792533357938131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,65535,0.05226133267084757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,131071,0.10522133111953735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,31,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,255,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,511,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1023,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1023,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,2047,0.012400000045696894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,2047,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,4095,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,4095,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,8191,0.016554666062196095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,8191,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,16383,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,32767,0.02743999908367793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,32767,0.027093333502610523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,131071,0.08906666437784831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,65535,0.0308693324526151
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,65535,0.03128000100453695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,3,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,131071,0.033045334120591484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,7,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,255,0.010389333590865135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,255,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,16383,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1023,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,2047,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,4095,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,4095,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,8191,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,8191,0.016879999389251072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,16383,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,16383,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,32767,0.021365332106749218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,32767,0.02125866711139679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,2047,0.011711999773979187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,65535,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,131071,0.032613334556420646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,131071,0.027306665976842243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,65535,0.02362666775782903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,63,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,511,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1023,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,2047,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,2047,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,4095,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,4095,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,8191,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,131071,0.027503999571005504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,511,0.011626667032639185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,16383,0.04116799930731455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,16383,0.03138133386770884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,32767,0.06075199941794077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,32767,0.052239999175071716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,8191,0.02312533309062322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,65535,0.10283733407656352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,65535,0.08810133735338847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,15,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,255,0.010133333504199982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,255,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1023,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,2047,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,131071,0.18931732575098673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,4095,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,4095,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,8191,0.017397332936525345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,16383,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,16383,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,32767,0.022389332453409832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,32767,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,131071,0.15899200240770975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,65535,0.024256000916163128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,131071,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,131071,0.029461334149042766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,8191,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,7,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,7,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,65535,0.022384000321229298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,511,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,511,0.01221866657336553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1023,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,2047,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,4095,0.021477334201335907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,4095,0.024133334557215374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,8191,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,8191,0.031285333136717476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,16383,0.060234665870666504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,16383,0.053946668903032936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,2047,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,32767,0.10234133402506511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,7,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,7,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,32767,0.08860266208648682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,65535,0.1851253310839335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,31,0.01173866664369901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,63,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,127,0.011973333855470022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,127,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,255,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,65535,0.15875200430552164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,511,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,511,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1023,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1023,0.01580799991885821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,2047,0.023872000475724537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,2047,0.023226665953795116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,4095,0.041034666200478874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,4095,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,15,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,8191,0.06252799928188324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,8191,0.05446400245030721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,3,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,16383,0.10409599542617798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,7,0.015589332828919092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,7,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,15,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,31,0.016794666647911072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,31,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,16383,0.08900266885757446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,63,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,15,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,63,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,255,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,255,0.014271999398867289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,511,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,3,0.013514666507641474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,511,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1023,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1023,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,127,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,2047,0.04116799930731455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,2047,0.031386665999889374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,4095,0.06242666641871134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,4095,0.05416533350944519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,7,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,8191,0.105621337890625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,7,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,63,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,63,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,255,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,255,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,511,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1023,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,2047,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,8191,0.08894399801890056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,2047,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,4095,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,4095,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,8191,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,8191,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,16383,0.019498666127522785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,16383,0.019007999449968338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,32767,0.02141333371400833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,65535,0.025216000775496166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,65535,0.025125332176685333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,32767,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,131071,0.044069334864616394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,131071,0.03363200028737386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,3,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,7,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,7,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,15,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,15,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,31,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,31,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,63,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,3,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,63,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,127,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,127,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,255,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,255,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,511,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,511,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1023,0.03573866685231527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1023,0.029653333127498627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,2047,0.05835199852784475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,2047,0.05151999990145365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,4095,0.09713600079218547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1,0.02537599951028824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1,0.023525332411130268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,3,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,3,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,7,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,4095,0.08305599788824718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,7,0.023717333873112995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,15,0.02515733242034912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,15,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,31,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,63,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,63,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,127,0.025514667232831318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,127,0.023061332603295643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,255,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,31,0.025237334271272022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,255,0.024725332856178284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,511,0.03170666595300039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1023,0.05614933371543884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1023,0.05013333261013031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,3,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,3,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,15,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,127,0.010357333347201347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,127,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,511,0.037989333271980286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,2047,0.09852799773216248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,2047,0.08492799599965413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,2047,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,4095,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,4095,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,8191,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,8191,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,16383,0.021013334393501282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,16383,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,32767,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,32767,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,65535,0.03982933362325033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,65535,0.03176533430814743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,131071,0.06214400132497152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,131071,0.05379199981689453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,31,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,255,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,2047,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,4095,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,8191,0.023290666441122692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,8191,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,16383,0.0401706670721372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,16383,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,511,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,32767,0.0613013356924057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,32767,0.039936001102129616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,65535,0.1023466686407725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,15,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,31,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,31,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,65535,0.059418668349583946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,63,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,511,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,511,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1023,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,2047,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,131071,0.10131200154622395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,4095,0.012725333372751871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,8191,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,8191,0.017103999853134155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,16383,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,16383,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,32767,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,131071,0.18515199422836304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,65535,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,4095,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,65535,0.029215998947620392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,32767,0.027802666028340656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,15,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,31,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,31,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,131071,0.03355200091997782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,511,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,2047,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,2047,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,4095,0.01350933313369751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,4095,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,8191,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,8191,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,16383,0.02170666555563609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,16383,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,131071,0.03562133262554804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,32767,0.025061334172884624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,32767,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,65535,0.027679999669392902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,131071,0.031717332700888314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,131071,0.027701333165168762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,255,0.011701333026091257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,511,0.011578666667143503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1023,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,2047,0.017477333545684814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,2047,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,65535,0.02405333270629247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,4095,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,4095,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,15,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,8191,0.023797333240509033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,16383,0.06039999922116598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,16383,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,32767,0.10249066352844238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,8191,0.039887999494870506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,65535,0.18369599183400473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,32767,0.05901333192984263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,7,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,131071,0.3484479983647664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1023,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,2047,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,4095,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,8191,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,8191,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,16383,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,65535,0.0997973382472992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,16383,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,32767,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,131071,0.18102933963139853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,65535,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,65535,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,131071,0.04864533245563507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1,0.012159999459981918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,32767,0.02350933353106181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,131071,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,3,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,3,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1023,0.014533333480358124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1023,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,2047,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,2047,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,4095,0.039488000174363456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,4095,0.0236160010099411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,8191,0.0603413333495458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,16383,0.10170666376749675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,16383,0.05971199770768484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,511,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,32767,0.18265599012374878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,32767,0.09909866253534953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,8191,0.039359999199708305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,3,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,3,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,65535,0.3457440137863159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,7,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,15,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,15,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,31,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,31,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,63,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,63,0.011781333635250727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,127,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,127,0.012319999436537424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,255,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,255,0.011514666179815928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,511,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1023,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1023,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,2047,0.03972800076007843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,2047,0.02347733328739802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,4095,0.061205332477887474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,4095,0.040448000033696495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,511,0.015354666858911514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,8191,0.10386133193969727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,8191,0.060746664802233376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1,0.013728000223636627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,16383,0.1850186586380005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,16383,0.10329066713651021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,65535,0.1794346570968628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,3,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,7,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,7,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,15,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,31,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,31,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,3,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,63,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,63,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,127,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,127,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,255,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,255,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,511,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,511,0.017477333545684814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1023,0.03531199942032496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1023,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,2047,0.0621919979651769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,2047,0.040192000567913055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,4095,0.10481599966684978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,15,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,63,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,8191,0.18645866711934408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,255,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,511,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,511,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1023,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,4095,0.06251200040181477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,2047,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,4095,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,4095,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,8191,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,8191,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,8191,0.10513066252072652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,16383,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,16383,0.019386666516462963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,32767,0.023711999257405598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,32767,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,65535,0.04311466713746389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,65535,0.025818665822347004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,2047,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1,0.019226666539907455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,131071,0.0639466643333435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,131071,0.04394666850566864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1023,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,3,0.019946667055288952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,3,0.019146667172511418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,7,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,7,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,15,0.019952000429232914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,15,0.018053332964579265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,31,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,31,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,63,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,63,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,127,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,127,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1,0.017727999637524288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,255,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,511,0.033333333830038704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,255,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1023,0.03355200091997782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1023,0.052239999175071716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,2047,0.09711999694506328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,4095,0.17520000537236533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,2047,0.05817066629727682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,4095,0.09564800063769023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1,0.029333333174387615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1,0.025242666403452556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,3,0.02934933453798294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,511,0.023498666783173878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,3,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,7,0.027903998891512554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,7,0.0259253333012263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,15,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,31,0.028970666229724884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,31,0.025429333249727886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,63,0.02759466568628947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,63,0.02518933266401291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,127,0.029258665939172108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,15,0.02977599948644638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,127,0.028938665986061096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,255,0.0331839993596077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,255,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,511,0.053930665055910744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,511,0.03716266651948293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1023,0.0913759966691335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1023,0.054560000697771706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,7,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,2047,0.1743839979171753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,31,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,511,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,511,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1023,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,2047,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,2047,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,2047,0.09726400176684062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,4095,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,8191,0.019573333362738293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,16383,0.02380266785621643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,16383,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,4095,0.01886933296918869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,8191,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,32767,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,65535,0.06211733321348826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,65535,0.04121600091457367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,32767,0.0420959989229838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1,0.04605866471926371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1,0.037978666524092354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1,0.03783999880154928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,3,0.04571733375390371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,131071,0.06234666705131531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,3,0.037946666280428566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,3,0.0376800000667572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,7,0.047456001242001854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1,0.04618666569391886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,7,0.03993066648642222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,7,0.03996799886226654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,15,0.04786666731039683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,15,0.04790399968624115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,15,0.04182399809360504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,15,0.04166933397452036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,31,0.05820799867312113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,31,0.05783466498057047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,31,0.0516480008761088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,131071,0.10082667072614034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,31,0.05193600058555603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,63,0.05807999769846598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,63,0.051872000098228455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,3,0.04565866788228353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,63,0.051781331499417625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,127,0.05821333328882853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,127,0.05852266649405161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,127,0.050383999943733215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,127,0.051962668697039284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,255,0.0582826683918635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,255,0.06881600121657054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,255,0.05018133421738943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,255,0.0602400004863739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,511,0.05801066756248474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,511,0.09941866993904114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,511,0.050474668542544045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,511,0.09330133597056071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1023,0.05825600028038025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1023,0.16084800163904825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1023,0.05016533533732096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1023,0.15454399585723877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,7,0.04814399778842926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,2047,0.28990399837493896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,2047,0.05186133086681366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,2047,0.29176000754038495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,4095,0.05827199916044871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,4095,0.5403786500295004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,2047,0.05795200169086456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1,0.045754666129748024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1,0.04577599962552389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,63,0.05675200124581655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1,0.03797333439191183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,3,0.04618666569391886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,3,0.045935998360315956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,3,0.03813866774241129
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,3,0.03824000060558319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,4095,0.05258666475613912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,7,0.048154667019844055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,7,0.04031999905904134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1,0.0376800000667572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,15,0.04785066843032837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,15,0.0481333335240682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,15,0.041589332123597465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,15,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,31,0.05598400036493937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,31,0.056176001826922096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,31,0.050069332122802734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,31,0.04981866478919983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,4095,0.535045345624288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,63,0.05594133337338766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,7,0.04028266668319702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,63,0.04997866849104563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,63,0.052522664268811546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,127,0.05881600081920624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,127,0.05208000044027964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,127,0.05213866631189982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,255,0.0584799995024999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,7,0.04752000172932943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,255,0.06841066479682922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,255,0.05188799897829691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,255,0.06208533545335134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,511,0.058464000622431435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,511,0.09941333532333374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,511,0.05175999800364176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,63,0.05788800120353699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1023,0.05846933523813883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,127,0.05782400071620941
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1023,0.051327998439470925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1023,0.15477333466211954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,2047,0.05824000140031179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,2047,0.2896266579627991
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,2047,0.281877338886261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,4095,0.05840000013510386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,511,0.09329066673914592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,4095,0.5428640047709147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1023,0.16077333688735962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1,0.04563733438650767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,4095,0.5335786739985148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1,0.03745066622893015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1,0.038015998899936676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,3,0.04603200157483419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,3,0.04602666695912679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,3,0.04161066561937332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,3,0.0378506655494372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,7,0.047872001926104225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,4095,0.05232533315817515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,7,0.039861333866914116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1,0.04571199913819631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,2047,0.05212800204753876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,15,0.04797866443792979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,15,0.04134399940570196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,15,0.04174399872620901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,31,0.05667200187842051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,31,0.05705066521962484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,31,0.049957334995269775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,7,0.0476800004641215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,31,0.05035733183224996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,63,0.05805333455403646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,63,0.057664001981417336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,63,0.050154666105906166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,63,0.05034666756788889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,127,0.05789333085219065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,127,0.05818133552869161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,127,0.050186668833096824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,127,0.0516480008761088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,255,0.057914664347966514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,255,0.0668693333864212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,255,0.050474668542544045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,255,0.06033066908518473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,511,0.05788800120353699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,511,0.09925867120424907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,511,0.04995200037956238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,511,0.09299199779828389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1023,0.05811200042565664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,7,0.039808000127474465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1023,0.16076800227165222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1023,0.051962668697039284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1023,0.15468266606330872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,2047,0.05842133363087972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,2047,0.05203199883302053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,2047,0.2837066650390625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,4095,0.056159997979799904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,4095,0.5459893147150675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,4095,0.049914668003718056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,2047,0.2898400028546651
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,4095,0.5316533247629801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,3,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,3,0.013978666315476099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,3,0.013807999591032663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,7,0.014074667046467463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,7,0.013701333353916803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,7,0.014229333649079004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,7,0.013621332744757334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,15,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,15,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,3,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,31,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,31,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,31,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,31,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,63,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,63,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,15,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,63,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,127,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,15,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,127,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,127,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,255,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,255,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,255,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,255,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,511,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,511,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,511,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1023,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,127,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1023,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1023,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,2047,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,2047,0.027077332139015198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,511,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,2047,0.015872000406185787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,2047,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,4095,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,4095,0.042853335539499916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,4095,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,4095,0.03545066714286804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1,0.00884799969693025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,15,0.04771733283996582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,3,0.008816000074148178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,3,0.008805333326260248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,3,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,3,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,7,0.00884799969693025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,7,0.009039999917149544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1023,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,15,0.009541333342591921
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,15,0.009375999992092451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,31,0.008922666932145754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,31,0.008789333204428354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,31,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,31,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,63,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,127,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,255,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,255,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,511,0.01947733387351036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,511,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,511,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1023,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1023,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1023,0.023050665855407715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1023,0.02889599899450938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,2047,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,2047,0.029370665550231934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,2047,0.023077333966890972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,2047,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,4095,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1,0.009658666948477427
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,4095,0.03996799886226654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,4095,0.023578666150569916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,4095,0.04173333446184794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,8191,0.062447999914487205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,8191,0.023077333966890972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,8191,0.06009600063165029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,16383,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,16383,0.10712533195813496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,16383,0.023215999205907185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,16383,0.09708266456921895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1,0.008837333569924036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,3,0.008821333448092142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,3,0.009301333377758661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,7,0.00933333362142245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,7,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1,0.00903466654320558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,15,0.008954666554927826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,15,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,31,0.009050666665037474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,31,0.00921066664159298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,7,0.009039999917149544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,63,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,63,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,15,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,127,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,127,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,255,0.010474666953086853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,511,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,511,0.021253332495689392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,511,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1023,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1023,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,8191,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1023,0.02717333287000656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,2047,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,2047,0.02942933390537898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,2047,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,2047,0.031530665854612984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,511,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,4095,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,4095,0.0397173340121905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,4095,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,4095,0.040965333580970764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,8191,0.019386666516462963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1023,0.025381334125995636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,8191,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,8191,0.0601440022389094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,255,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,16383,0.10731732845306396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,16383,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1,0.009392000113924345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1,0.009029333169261614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,3,0.009418666362762451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,8191,0.062181333700815834
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,7,0.009002666920423508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,7,0.009029333169261614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,16383,0.09738666812578838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,15,0.008757333581646284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,15,0.008863999818762144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,15,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,3,0.009066666786869368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,31,0.009674666449427605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,31,0.009365333244204521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,31,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,63,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,127,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,15,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,16383,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,127,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,255,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,255,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,511,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,511,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,511,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1023,0.019546666493018467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1023,0.023733332753181458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1023,0.023525332411130268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1023,0.027514666318893433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,2047,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,2047,0.029626667499542236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,2047,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,2047,0.03153600047032038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,4095,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,4095,0.04117333392302195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,4095,0.023711999257405598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,4095,0.04182399809360504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,8191,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,8191,0.06444266438484192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,8191,0.023546665906906128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,8191,0.06075199941794077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,16383,0.10761066277821858
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,16383,0.02316266546646754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,3,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,16383,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,7,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,16383,0.09760533769925435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,15,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,511,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,511,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1023,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1023,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,2047,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,2047,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,2047,0.012469333906968435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,2047,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,4095,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,4095,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,4095,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,4095,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,8191,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,8191,0.016384000579516094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,8191,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,8191,0.016666666915019352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,16383,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,16383,0.018677332748969395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,16383,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,16383,0.018042666216691334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1,0.01351999988158544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,3,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,3,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,3,0.00914666677514712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,3,0.009178666397929192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,7,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,7,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,7,0.010410666465759277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,15,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,15,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,15,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,15,0.010431999961535135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,31,0.014773332824309668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,31,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,63,0.01532799998919169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,63,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,63,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,127,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,127,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,127,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,255,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,31,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,255,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,255,0.012597333639860153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,511,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,511,0.015471999843915304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1023,0.023562667270501454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1023,0.013744000345468521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1023,0.019882666567961376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,255,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,2047,0.01603200038274129
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,2047,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,2047,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,2047,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,4095,0.015829333414634068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,4095,0.05209066470464071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,4095,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,4095,0.04887466629346212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1023,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,8191,0.0930613378683726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,8191,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,8191,0.08699733018875122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,16383,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,16383,0.17075733343760172
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,16383,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,511,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1,0.009872000043590864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1,0.009775999933481216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,3,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,3,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,3,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,7,0.013610667238632837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,7,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,7,0.010309333602587381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,15,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,15,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,15,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,31,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,31,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,63,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,63,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,63,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,127,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,127,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,127,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,127,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,255,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,255,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,255,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,255,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,511,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,511,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,8191,0.015599999576807022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,511,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1023,0.01545599972208341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1023,0.02314666658639908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1023,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1023,0.01958400011062622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,16383,0.16480533281962076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,2047,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,2047,0.03326933334271113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,2047,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,2047,0.029637334247430164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,4095,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,4095,0.05177066723505656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,4095,0.04804799954096476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,8191,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,8191,0.09295466542243958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,8191,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,8191,0.0869760016600291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,16383,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,16383,0.1707893411318461
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,16383,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,63,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,16383,0.1644000013669332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1,0.008986666798591614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,3,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,3,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,3,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,7,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,7,0.012186666329701742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,15,0.013776000589132309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,15,0.013354666531085968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,15,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,31,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,31,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,63,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,63,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,63,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,127,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,127,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,127,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,255,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,255,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,255,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,7,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,511,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,511,0.014725333700577417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1023,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1023,0.023024000227451324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1023,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,2047,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,2047,0.033376000821590424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,2047,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,2047,0.02940800040960312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,4095,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,4095,0.052149335543314614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,4095,0.04943466683228811
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,8191,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,8191,0.09297066926956177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,8191,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,8191,0.08721599976221721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,16383,0.015509333461523056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,16383,0.171450674533844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,127,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1,0.011648000528415045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,3,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,511,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,15,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,31,0.011610666910807291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,31,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,63,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,16383,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,127,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,16383,0.16449066996574402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,2047,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,2047,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,2047,0.012117333710193634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,2047,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,4095,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,4095,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,4095,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,8191,0.01250133290886879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,511,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,8191,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,8191,0.011770666887362799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,8191,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,16383,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,16383,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,16383,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,16383,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1,0.08486933509508769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1,0.06975999971230824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1,0.07045866549015045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,3,0.08505066235860188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,3,0.08517332871754964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,3,0.07075733443101247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,3,0.0698880006869634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,7,0.08773866295814514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,7,0.08717333277066548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,7,0.07260799904664357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,7,0.0724533349275589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,15,0.08930133779843648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,15,0.08893332878748576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,15,0.07489599784215291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,15,0.07473599910736084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1,0.08515733480453491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,31,0.10711999734242757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,31,0.09516800443331401
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,31,0.09316800038019817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,63,0.10761599739392598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,63,0.1074186662832896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,63,0.09508267045021057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,63,0.0955466628074646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,127,0.10922666390736897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,127,0.09577600161234538
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,127,0.0953653355439504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,255,0.10955733060836792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,255,0.12805333733558655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,255,0.09591999650001526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,31,0.10710933804512024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,511,0.10962667067845662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,511,0.19035732746124268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,511,0.0958186686038971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,511,0.1771519978841146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1023,0.10971732934315999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,127,0.1076639990011851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1023,0.09547199805577596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1023,0.30083733797073364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,255,0.11558399597803752
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,2047,0.5671146710713705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,2047,0.5501279830932617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1,0.08457600076993306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1,0.09117333094278972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,2047,0.10962667067845662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1,0.0697920024394989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,3,0.08539733290672302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,3,0.0851200024286906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,3,0.07049066821734111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,3,0.07046400010585785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,7,0.08682133754094441
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,7,0.08695466319719951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,7,0.07243200143178304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,7,0.07231999933719635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,15,0.08910399675369263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,15,0.08865599830945332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,15,0.07439466814200084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,15,0.07493333518505096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,31,0.1071626643339793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,31,0.10710400342941284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,31,0.09516800443331401
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,31,0.09431466460227966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,2047,0.09530133008956909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,63,0.10753066341082256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,63,0.09540266791979472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,63,0.09527466694513957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1023,0.3125386635462443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,127,0.10937600334485371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1,0.06922133266925812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,127,0.09502399961153667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,63,0.10750400026639302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,255,0.12761599818865457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,255,0.09512000282605489
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,255,0.11474666992823283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,127,0.1095306674639384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,511,0.19131733973821005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,127,0.09520000219345093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,511,0.0969493289788564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,511,0.17709332704544067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1023,0.1095199982325236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,255,0.10949333508809407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1023,0.09710933764775594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1023,0.30135466655095416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,2047,0.10950400431950887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,2047,0.09540800253550212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,2047,0.5648906628290812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,511,0.10964799920717876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1,0.08497066299120586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1,0.08483200271924336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,2047,0.5559146801630656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1,0.06838933130105336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,3,0.08480000495910645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1023,0.31250667572021484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,3,0.07072533170382182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,3,0.07080533107121785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,7,0.0872373382250468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,7,0.0869706670443217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1,0.07282133400440216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,7,0.07257600128650665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,15,0.08912533521652222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,3,0.08893332878748576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,15,0.0747573326031367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,15,0.07486933469772339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,31,0.1072213351726532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,31,0.10752000411351521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,7,0.07257600128650665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,63,0.10760000348091125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,63,0.10735467076301575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,15,0.08901333808898926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,63,0.09530666470527649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,127,0.10912000139554341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,127,0.10941867033640544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,127,0.09507733583450317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,31,0.09448533256848653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,255,0.1093280017375946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,255,0.1279146671295166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,63,0.09488532940546672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,255,0.11370666821797688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,511,0.10955199599266052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,511,0.18968532482783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,511,0.09477866689364116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,31,0.09504533807436626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,511,0.17787732680638632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1023,0.10937600334485371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1023,0.0955466628074646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,255,0.09513599673906963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1023,0.302181343237559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,2047,0.10945066809654236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,2047,0.09529067079226176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,2047,0.5750186840693156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,2047,0.5540053447087606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1023,0.31268266836802167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,3,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,3,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,3,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,3,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,7,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,7,0.016042667130629223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,7,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,7,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,15,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,15,0.015423999478419622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,15,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,127,0.09500267108281453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,31,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,31,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,31,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,63,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,63,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,63,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,63,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,127,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,127,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,127,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,127,0.015322666615247726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,255,0.01551466683546702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,255,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,255,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,255,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,15,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,511,0.018954666952292126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,511,0.016864000509182613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,511,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1023,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1023,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1023,0.01722666621208191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1023,0.02367466688156128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,2047,0.018911999960740406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,2047,0.04366933306058248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,2047,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,2047,0.035973332822322845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,3,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,3,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,3,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,3,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,7,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,7,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,7,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,7,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1,0.015509333461523056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,15,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,15,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,15,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,31,0.014010666559139887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,31,0.014111999422311783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,31,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,31,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,63,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,63,0.01952533299724261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,63,0.019813333948453266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,127,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,127,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,127,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,127,0.01966933285196622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,255,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,255,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,255,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,255,0.020703999946514767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,511,0.01653333380818367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,511,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,511,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,511,0.023029332359631855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1023,0.015685333559910457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1023,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1023,0.03355200091997782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,2047,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,2047,0.05180266499519348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,63,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,2047,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,2047,0.052144000927607216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,4095,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,4095,0.09107200304667155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,4095,0.01923199991385142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,4095,0.08877333005269368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1023,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,8191,0.1729066570599874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,8191,0.16263467073440552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,16383,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,16383,0.3373013337453206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,511,0.019621333728233974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,16383,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,8191,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,16383,0.3128053347269694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1,0.013487999637921652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,3,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,3,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,3,0.015439999600251516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,3,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,7,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,7,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,7,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,7,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,15,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,15,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,15,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,31,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,31,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,31,0.016757333030303318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,31,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,63,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,63,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,63,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,63,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,127,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,127,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,127,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,127,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,255,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,255,0.01553600033124288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,255,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,255,0.019445333629846573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,511,0.021583999196688335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,511,0.01953599974513054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,511,0.022917332748572033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1023,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1023,0.0314026673634847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1023,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1023,0.0334346666932106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,2047,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,2047,0.05226133267084757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,2047,0.01950399950146675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,8191,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,4095,0.09085866808891296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,4095,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,4095,0.08907199899355571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,8191,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,511,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,8191,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,2047,0.051962668697039284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,8191,0.16305599610010782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,16383,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,16383,0.33984533945719403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,16383,0.31226666768391925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1,0.014639999717473984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,4095,0.014837333311637243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,3,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,16383,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,3,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,3,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,7,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,7,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,7,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,7,0.015487999965747198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,15,0.013487999637921652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,15,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,15,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,3,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,15,0.01532799998919169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,31,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,31,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,31,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,31,0.016730666160583496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,63,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,63,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,8191,0.17291200160980225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,63,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,127,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,127,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,127,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,255,0.015669333438078564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,255,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,255,0.019695999721686046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,255,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,511,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,511,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,511,0.019626667102177937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,511,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1023,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1023,0.03125333289305369
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1023,0.020560000091791153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1023,0.03181866556406021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,2047,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,2047,0.051781331499417625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,2047,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,2047,0.05147199829419454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,4095,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,4095,0.09129066268603007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,4095,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,4095,0.08867733677228291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,8191,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,8191,0.17446933190027872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,8191,0.16267733772595724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,63,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,127,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,16383,0.3407893180847168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,16383,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,16383,0.31226134300231934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,8191,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,3,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,16383,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,7,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,15,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,15,0.012037333101034164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,31,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,31,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,63,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,511,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,511,0.012703999876976013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1023,0.011695999652147293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,2047,0.012730666746695837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,2047,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,2047,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,2047,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,4095,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,4095,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,4095,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,4095,0.016165333489576977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,8191,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,8191,0.01978133370478948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,8191,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,8191,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,16383,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,16383,0.02554133286078771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,16383,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,16383,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1,0.16291733582814535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1,0.16289599736531576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1,0.13205867012341818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1,0.13179733355840048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,3,0.16294399897257486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,3,0.1628266672293345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,3,0.1318186620871226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,3,0.13195199767748514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,7,0.16870933771133423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,7,0.16894932587941489
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,7,0.13737066586812338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,7,0.13623467087745667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,15,0.17097065846125284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,15,0.1718133290608724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,15,0.1427839994430542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,31,0.20588266849517822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,31,0.2063466707865397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,31,0.18117332458496094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,31,0.18132267395655313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,63,0.2077653408050537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,63,0.20790932575861612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,63,0.18339200814565024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,63,0.1813066601753235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,127,0.20972800254821777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,127,0.2097813288370768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,15,0.14272532860438028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,127,0.1833440065383911
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,255,0.21181867520014444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,255,0.2469866673151652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,255,0.1832480033238729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,255,0.22200000286102295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,511,0.2119306723276774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,511,0.37246934572855633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1023,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,511,0.3460266590118408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1023,0.21193599700927734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1023,0.1832159956296285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1023,0.617248018582662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1,0.1628426710764567
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1023,0.5911466677983602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1,0.1625706652800242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1,0.13174933195114136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,511,0.185098667939504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,3,0.17523199319839478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,3,0.16295466820398966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,3,0.13221866885821024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,3,0.13214932878812155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,7,0.16686934232711792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,7,0.1688213348388672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,7,0.1362933317820231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,7,0.13808533549308777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,15,0.1709866722424825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,15,0.1711840033531189
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,15,0.14222932855288187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,15,0.14265066385269165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,31,0.2059626579284668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,31,0.2077653408050537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,31,0.1808639963467916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,31,0.1813066601753235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,63,0.20800000429153442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,63,0.20827200015385947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,63,0.18134399255116782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,63,0.18324265877405801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,127,0.2100480000178019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,127,0.21211200952529907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,127,0.18349866072336832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,127,0.18341867129007974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,255,0.21093332767486572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,255,0.2480213244756063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,255,0.18352532386779785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,255,0.2222879926363627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,511,0.21196266015370688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,511,0.3720746835072835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,511,0.3475840091705322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1,0.13211733102798462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1023,0.21224000056584677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,127,0.18331732352574667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1023,0.5919359922409058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,511,0.18329066038131714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1,0.16295466820398966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1,0.16265599926312765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1,0.13217066725095114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1,0.13184000054995218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,3,0.16263467073440552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,3,0.1627893348534902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,3,0.13197867075602213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1023,0.18332799275716147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,7,0.16677866379419962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,7,0.16866666078567505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,7,0.13774933417638144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,7,0.13784533739089966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,15,0.17093332608540854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,15,0.17096533377965292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,15,0.1423413356145223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,15,0.14231466253598532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,31,0.20600533485412598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,31,0.20618132750193277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,31,0.18134933710098267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,31,0.181002676486969
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,63,0.20791999499003092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,3,0.1320319970448812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,63,0.18291733662287393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,63,0.1820639967918396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,127,0.21088000138600668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,127,0.18419732650121054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,127,0.18481600284576416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1023,0.6157333453496298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,255,0.24713067213694254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,255,0.1835199991861979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,127,0.209824005762736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,255,0.22234133879343668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,511,0.21182932456334433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,511,0.3712799946467082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,511,0.18301333983739218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,511,0.3460853497187297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1023,0.21178134282430014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1023,0.1832480033238729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1023,0.6337600151697794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1023,0.5923146804173788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1,0.020410666863123577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,3,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,255,0.21134400367736816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,3,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,3,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,3,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,7,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,7,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,7,0.020319999506076176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,7,0.019578666736682255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,15,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,15,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,15,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,31,0.02093333254257838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,31,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,31,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,31,0.020319999506076176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,63,0.20805333058039346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,63,0.02102400114138921
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,63,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,63,0.019626667102177937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,127,0.01941866676012675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,127,0.021002667645613354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,127,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,127,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,255,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,255,0.02089066555102666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,255,0.020975999534130096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,255,0.01960533360640208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,511,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,511,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,511,0.02141333371400833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,511,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1023,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1023,0.03979733337958654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1023,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1023,0.033514666060606636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,1,0.3166559934616089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,1,0.31646400690078735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,1,0.2550719976425171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,1,0.25473066171010333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,3,0.3165493408838908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,3,0.316922664642334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,3,0.2549333373705546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,3,0.25464532772699994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,7,0.3269760012626648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,7,0.3277119994163513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,15,0.0210506667693456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,7,0.2651679913202922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,7,0.2655946612358093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,15,0.3346720139185588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,15,0.334879994392395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,15,0.2775413393974304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,63,0.0210506667693456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,31,0.404639999071757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,31,0.4044586817423503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,31,0.3537813425064087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,31,0.35367465019226074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,63,0.40858666102091473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,63,0.40958933035532635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,63,0.35706667105356854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,63,0.35710398356119794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,127,0.41462401549021405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,127,0.4238933324813843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,127,0.3591466744740804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,15,0.27747732400894165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,255,0.41444265842437744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,255,0.48683734734853107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,255,0.3595893383026123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,255,0.43510933717091876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,1,0.31730665763219196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,1,0.25646400451660156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,127,0.3612906535466512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,1,0.2571306626001994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,3,0.31865066289901733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,3,0.3184853394826253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,3,0.2569226622581482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,3,0.25704532861709595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,7,0.32898133993148804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,7,0.32893333832422894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,7,0.2669066588083903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,7,0.26714666684468585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,15,0.3348960081736247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,15,0.3349066575368245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,15,0.2777013381322225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,1,0.3174399932225545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,15,0.2781066695849101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,31,0.4065759976704915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,31,0.4050559997558594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,31,0.35410133997599286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,31,0.3537973165512085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,63,0.4106666644414266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,63,0.4108266830444336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,63,0.3582079807917277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,63,0.35728001594543457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,127,0.41490666071573895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,127,0.41298667589823407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,127,0.3598933219909668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,127,0.3596320152282715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,255,0.41475733121236164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,255,0.4869333505630493
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,255,0.35946667194366455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,255,0.4328746795654297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,1,0.31852267185846966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,1,0.317413330078125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,1,0.25760533412297565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,3,0.31843199332555133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,3,0.3184373378753662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,3,0.2571679949760437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,3,0.25730133056640625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,7,0.32822932799657184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,7,0.32655467589696247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,7,0.2674720088640849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,7,0.2672906716664632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,15,0.33479468027750653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,15,0.3347359895706177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,15,0.27802133560180664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,15,0.2773386637369792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,1,0.2604373296101888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,31,0.4052480061848958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,31,0.4050026734670003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,31,0.3531999985376994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,31,0.35362664858500165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,63,0.4094933271408081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,63,0.4102240006128947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,63,0.35736000537872314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,127,0.41431466738382977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,127,0.4129386742909749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,127,0.3598080078760783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,127,0.35924800237019855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,255,0.4144586722056071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,255,0.5272106726964315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,255,0.4355893135070801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,1,0.031514666974544525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,63,0.3575253486633301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,1,0.02937600016593933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,1,0.029370665550231934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,3,0.031504000226656594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,3,0.03155199935038885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,3,0.029520000020662945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,3,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,7,0.03156800071398417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,7,0.03133866687615713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,7,0.029530666768550873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,7,0.03070933371782303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,15,0.03139200061559677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,15,0.03127466638882955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,15,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,15,0.029493334392706554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,31,0.031632001201311745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,31,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,31,0.029834667841593426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,31,0.02958933264017105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,63,0.031541332602500916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,63,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,63,0.02951466788848241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,63,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,127,0.03148799886306127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,127,0.03155199935038885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,127,0.02956266701221466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,127,0.02979733298222224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,255,0.033359999457995095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,255,0.03189333279927572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,255,0.03153600047032038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,255,0.029578665892283123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,255,0.35947732130686444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,1,0.03139200061559677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,1,0.6282986799875895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,1,0.625930666923523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,1,0.5021013418833414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,1,0.5174346764882406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,3,0.6262240012486776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,3,0.6279679934183756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,3,0.50054931640625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,3,0.5017600059509277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,7,0.6484479904174805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,7,0.6339946587880453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,7,0.5244266589482626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,7,0.5246986548105875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,15,0.6610293388366699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,15,0.6601759990056356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,15,0.5490133364995321
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,15,0.547823985417684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,31,0.8022399743398031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,31,0.8041280110677084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,31,0.6991466681162516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,31,0.6996373335520426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,63,0.8104906876881918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,63,0.8114453156789144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,63,0.7057867050170898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,63,0.7073386510213217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,127,0.8182559808095297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,127,0.8191626866658529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,127,0.7105387051900228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,127,0.7115680376688639
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,1,0.6259520053863525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,1,0.6272373199462891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,1,0.5031840006510416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,1,0.5042399962743124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,3,0.6286346515019735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,3,0.632207989692688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,3,0.5042879978815714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,3,0.5048426787058512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,7,0.6468213399251302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,7,0.6480799913406372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,7,0.5256426731745402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,7,0.5253119866053263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,15,0.6604533195495605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,15,0.6606239875157675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,15,0.5482826630274454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,31,0.8047466278076172
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,31,0.8044319947560629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,31,0.6997546354929606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,31,0.7003893057505289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,63,0.8125332991282145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,63,0.8121493657430013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,15,0.5478560129801432
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,63,0.7070079644521078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,63,0.7062239646911621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,127,0.8200053373972574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,127,0.8191146850585938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,127,0.7119466463724772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,1,0.6263946692148844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,1,0.6273333231608073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,1,0.5085599819819132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,1,0.5068320035934448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,3,0.6497919956843058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,3,0.6279360055923462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,3,0.5073279937108358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,3,0.5078506469726562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,7,0.6483306487401327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,7,0.6468266646067301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,7,0.5251839955647787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,7,0.5273706515630087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,15,0.660431981086731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,15,0.6604373455047607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,15,0.5496746699015299
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,127,0.7110613187154134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,15,0.5480639934539795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,31,0.803610642751058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,31,0.8043466409047445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,31,0.6998506387074789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,63,0.8121066888173422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,63,0.8125973542531332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,63,0.7057387034098307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,63,0.7070026397705078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,127,0.8206506570180258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,127,0.8186079661051432
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,127,0.711616039276123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,1,0.05219733218352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,1,0.05221333106358846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,1,0.04978133241335551
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,1,0.04978133241335551
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,3,0.05227200190226237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,3,0.05235200126965841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,3,0.04996799925963084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,31,0.7010666529337565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,3,0.04881600042184194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,7,0.05237866441408793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,7,0.05273066461086273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,7,0.04974933465321859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,15,0.05212800204753876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,15,0.053210665782292686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,15,0.050010666251182556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,15,0.04987200101216634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,31,0.052069331208864846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,31,0.05195199946562449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,31,0.04961599906285604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,7,0.05016533533732096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,31,0.04984533290068308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,63,0.05272000034650167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,63,0.052789335449536644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,63,0.04987733562787374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,63,0.050016000866889954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,127,0.053818667928377785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,127,0.054085334142049156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,127,0.048197334011395775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,127,0.04994666576385498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1,0.015642666568358738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,3,0.018725333114465077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,3,0.018511999398469925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,3,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,3,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,127,0.7115039825439453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,7,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,7,0.015775999675194424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,15,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,15,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,15,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,31,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,31,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,31,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,7,0.018960000326236088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,63,0.01947733387351036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,7,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,15,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,63,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,127,0.02060266708334287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,127,0.01993600030740102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,127,0.017759999881188076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,127,0.01764800027012825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,255,0.019850666324297588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,255,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,63,0.01950399950146675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,63,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,511,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,511,0.03136533250411352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,511,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,511,0.029391999046007793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1023,0.02089600016673406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1023,0.0476746658484141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,31,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1023,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1023,0.04577066500981649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,2047,0.0207893339296182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,2047,0.08308800061543782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,2047,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,2047,0.08084266881148021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,4095,0.020853333175182343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,4095,0.15052800377209982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,4095,0.14832533399264017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,8191,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,8191,0.28628265857696533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,8191,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,255,0.02089600016673406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,8191,0.3097066680590312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,16383,0.021221332252025604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,4095,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,16383,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1,0.017477333545684814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,16383,0.5972799857457479
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,255,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,3,0.017583999782800674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,3,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,3,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,7,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,16383,0.5928959846496582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,7,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,15,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,15,0.017770666629076004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,15,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,15,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,31,0.019386666516462963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,31,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,31,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,31,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,63,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,7,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,63,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,63,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,3,0.01870399961868922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,127,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,127,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,127,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,255,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,7,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,255,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,63,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,511,0.020874666670958202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,511,0.03152533372243246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,127,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,255,0.023168000082174938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,511,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,255,0.02143999934196472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1023,0.01916266605257988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1023,0.04805333415667216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1023,0.04665066798528036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,2047,0.020954666038354237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,2047,0.08286933104197185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,2047,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,2047,0.08094933132330577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,4095,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,4095,0.15069333712259927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1023,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,4095,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,511,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,4095,0.1483680009841919
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,8191,0.0220266655087471
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,8191,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,16383,0.021029333273569744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,16383,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,16383,0.5906399885813395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,16383,0.5993226766586304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,3,0.017583999782800674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,8191,0.2858293255170186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,3,0.015392000476519266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,8191,0.28520532449086505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,7,0.015552000453074774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,7,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,15,0.017765333255132038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,15,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,15,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,3,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,15,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,31,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,31,0.019626667102177937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,31,0.017397332936525345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,31,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,63,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,7,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,63,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,63,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,127,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,127,0.019146667172511418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,127,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,127,0.018863999595244724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,255,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,255,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,255,0.021525333325068157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,511,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,511,0.03161599983771642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,511,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,7,0.019146667172511418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,63,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1023,0.02096533278624217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1023,0.04821333289146423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1023,0.0189280000825723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1023,0.046426668763160706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,2047,0.020997333029905956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,2047,0.08295999964078267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,255,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,2047,0.08101333181063335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,4095,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,4095,0.15057067076365152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,4095,0.14865600069363913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,511,0.02956799914439519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,8191,0.2892106572786967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,8191,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,2047,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,8191,0.28657599290211994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,16383,0.018522666146357853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,16383,0.6237599849700928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,8191,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,16383,0.6005119880040487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,3,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,3,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,63,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,255,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,255,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,511,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,511,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,511,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,511,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1023,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,2047,0.01762666677435239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,2047,0.017642666896184284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,4095,0.013568000247081121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,4095,0.019738666713237762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,4095,0.013760000467300415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1023,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,8191,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,8191,0.025573333104451496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,2047,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,16383,0.013605333864688873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,16383,0.04298666616280874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,16383,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,4095,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,16383,0.035317334036032356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,8191,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,8191,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,1,1.2448853651682537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,1,1.250106652577718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,1,0.9920799732208252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,1,0.9926933447519938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,3,1.2472000122070312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,3,0.9904053211212158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,3,0.9927840232849121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,7,1.2873600323994954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,7,1.288325309753418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,3,1.2455999851226807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,7,1.041376034418742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,15,1.3156426747639973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,7,1.0416959921518962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,15,1.089733362197876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,15,1.0892213185628254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,31,1.5668106079101562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,31,1.5986933708190918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,31,1.3936907450358074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,31,1.3929012616475422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,63,1.6171305974324544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,63,1.6160747210184734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,15,1.2865866820017497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,63,1.4053653081258137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,63,1.4062986373901367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,1,1.2458720207214355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,1,1.238213300704956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,1,0.9989279905954996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,1,0.998639980951945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,3,1.24673064549764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,3,1.2481333414713542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,3,1.0015626748402913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,7,1.2896426518758137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,7,1.2935787041982014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,7,1.0433653195699055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,3,1.000432014465332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,15,1.2833013534545898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,15,1.3166613578796387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,15,1.0890560150146484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,15,1.0923413435618083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,31,1.5998133023579915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,31,1.598218599955241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,31,1.3912372589111328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,31,1.393653392791748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,7,1.0425653457641602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,63,1.6169652938842773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,63,1.6156853040059407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,63,1.4070186614990234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,63,1.4070879618326824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,1,1.2472906907399495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,1,1.2456639607747395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,1,1.0081760088602703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,1,1.008464018503825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,3,1.2471413612365723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,3,1.2470026810963948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,3,1.0093226432800293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,7,1.2875946362813313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,7,1.2869226932525635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,7,1.045141299565633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,7,1.043514649073283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,3,1.0102826754252117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,15,1.3135840098063152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,15,1.3140052954355876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,15,1.090015967686971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,15,1.0914560159047444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,31,1.5994240442911785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,31,1.6008747418721516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,31,1.3931679725646973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,31,1.3929759661356609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,63,1.6315627098083496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,63,1.6238400141398113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,1,0.09515733520189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,63,1.405194600423177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,63,1.4073012669881184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,1,0.09518933296203613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,1,0.08731733759244283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,1,0.08891200025876363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,3,0.09520000219345093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,3,0.0956213374932607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,3,0.08709333340326945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,3,0.08699199557304382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,7,0.09511466821034749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,7,0.09526399771372478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,7,0.08693866928418477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,7,0.08692800005276997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,15,0.09518933296203613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,15,0.09577066699663798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,15,0.08675733208656311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,31,0.09506666660308838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,31,0.09506666660308838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,31,0.08693866928418477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,31,0.08691199620564778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,63,0.09530666470527649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,63,0.08687466382980347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,15,0.08726933598518372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,63,0.08691199620564778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,63,0.09515200058619182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,1,2.4829279581705728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,1,2.4830880165100098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,1,1.981877326965332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,1,1.9795680046081543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,3,2.4851840337117515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,3,2.480384031931559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,3,1.980613390604655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,3,1.9838666915893555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,7,2.573733329772949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,7,2.569002628326416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,7,2.0818613370259604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,7,2.08077335357666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,15,2.6339200337727866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,15,2.629711945851644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,15,2.1781439781188965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,15,2.1777599652608237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,31,3.2012853622436523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,31,3.1970933278401694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,31,2.7863146464029946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,31,2.785674730936686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,1,2.48633607228597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,1,2.0002454121907554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,1,2.000943978627523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,1,2.4884959856669107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,3,2.493130683898926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,3,2.4981600443522134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,3,2.003946622212728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,3,1.998154640197754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,7,2.5863893826802573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,7,2.5927947362264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,7,2.084730625152588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,7,2.0816052754720054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,15,2.6659520467122397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,15,2.6659413973490396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,15,2.1786826451619468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,15,2.182912031809489
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,31,3.2439359029134116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,31,2.787071863810221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,31,2.7940638860066733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,1,2.5129599571228027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,1,2.5077226956685386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,1,2.014666716257731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,31,3.2445281346639
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,3,2.523594697316488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,3,2.5095626513163247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,3,2.0245653788248696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,3,2.0264533360799155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,1,2.0234452883402505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,7,2.6355412801106772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,7,2.0976266860961914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,7,2.621093273162842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,7,2.101952075958252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,15,2.7743307749430337
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,15,2.7643200556437173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,15,2.204357306162516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,15,2.2127040227254233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,31,3.3162666956583657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,31,2.8389546076456704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,1,0.17819732427597046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,31,2.835023880004883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,1,0.17825067043304443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,1,0.1649386684099833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,1,0.1649333337942759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,3,0.17922665675481161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,3,0.17712533473968506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,3,0.16897066434224448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,3,0.16554666558901468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,7,0.17715734243392944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,7,0.1792746583620707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,31,3.3110294342041016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,7,0.16527466972668967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,7,0.16514133413632712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,15,0.17869865894317627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,15,0.16482667128245035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,15,0.16500799854596457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,31,0.17778132359186807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,31,0.1786880095799764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1,0.027893332143624622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1,0.02779199928045273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1,0.02386666586001714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1,0.023984000086784363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,31,0.16501866777737936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,3,0.027493332823117573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,3,0.02719466636578242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,3,0.023434666295846302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,3,0.023669332265853882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,7,0.02808533360560735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,7,0.028005334238211315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,7,0.0239680012067159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,7,0.023525332411130268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,15,0.02773866554101308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,31,0.16474133729934692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,15,0.023647998770078022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,15,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,31,0.031541332602500916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,31,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,31,0.02941333254178365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,63,0.03196266790231069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,63,0.031397332747777305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,63,0.029440000653266907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,63,0.029343999922275543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,15,0.027402666707833607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,127,0.03178666780392329
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,31,0.03245333333810171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,127,0.02937600016593933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,255,0.03183466692765554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,255,0.03782399992148081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,255,0.02951466788848241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,255,0.033904001116752625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,511,0.03160000095764796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,15,0.17923200130462646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,511,0.054117331902186074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,511,0.030063999195893604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1023,0.03387200087308884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1023,0.08718400200208028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1023,0.029834667841593426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1023,0.08504533767700195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,2047,0.03355200091997782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,2047,0.153738667567571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,2047,0.029535998900731403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,511,0.05012799799442291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,2047,0.1504800021648407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,127,0.02956799914439519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,4095,0.2868853410085042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,4095,0.2818880081176758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,8191,0.03330666571855545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,8191,0.5542720158894857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,8191,0.029279999434947968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,4095,0.03180266668399175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1,0.02737066646416982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,8191,0.5474720001220703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1,0.023770667612552643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,3,0.027514666318893433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,3,0.02743999908367793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,3,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,127,0.031658666829268135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,3,0.023285334308942158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,7,0.027072000006834667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,7,0.027808000644048054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,7,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,7,0.023445333043734234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,15,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,4095,0.02941333254178365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,15,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1,0.02731200059254964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,31,0.031850665807724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,31,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,31,0.029290666182835896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,31,0.02962133288383484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,63,0.03161599983771642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,63,0.031317333380381264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,63,0.029066666960716248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,63,0.029285334050655365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,127,0.03292799989382426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,127,0.03141333411137263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,127,0.031061333914597828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,127,0.029509333272775013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,255,0.032960000137488045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,255,0.037845333417256675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,255,0.029178666571776073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,255,0.033610666791598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,511,0.03336533407370249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,511,0.0540533314148585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,511,0.029232000311215717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,511,0.05096533397833506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1023,0.03338133295377096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1023,0.08705066641171773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1023,0.02920000006755193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1023,0.08298666775226593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,2047,0.033386667569478355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,2047,0.1525759994983673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,2047,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,2047,0.15036799510320029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,15,0.023290666441122692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,4095,0.2857973376909892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,4095,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,4095,0.2815840045611064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,8191,0.03332799921433131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,15,0.02752000093460083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,4095,0.03169066707293192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,8191,0.0296426663796107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1,0.02734400083621343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1,0.02370133250951767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,3,0.02754133443037669
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,3,0.024234667420387268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,3,0.02390933285156886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,7,0.02784000088771184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,8191,0.5522986650466919
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,7,0.027845333019892376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,7,0.023306667804718018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,7,0.023946667710940044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,15,0.02757333219051361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,15,0.027679999669392902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,15,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,15,0.02383466561635335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,31,0.032229334115982056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,31,0.029274667302767437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,31,0.02958400050799052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,63,0.03204799940188726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,63,0.031712000568707786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,63,0.029616000751654308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,63,0.029637334247430164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,127,0.03156266609827677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,3,0.027162666122118633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,127,0.029669334491093952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,127,0.029418667157491047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,255,0.033333333830038704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,255,0.03779733429352442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,255,0.029648000995318096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,255,0.03385599950949351
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,8191,0.5464906692504883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,511,0.05399466554323832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,127,0.03177600105603536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,511,0.0499839981396993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1023,0.0315786674618721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1023,0.08667199810345967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1023,0.02938666691382726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1023,0.0828906645377477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,2047,0.03178133318821589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,2047,0.15256533026695251
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,31,0.03154666721820831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,2047,0.029258665939172108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,2047,0.1502346694469452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,4095,0.03339199970165888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,4095,0.02932800104220708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,4095,0.2815786600112915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,8191,0.03333866596221924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,511,0.03374933451414108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,8191,0.029637334247430164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,511,0.02923733244339625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,3,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,15,0.01249066616098086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,15,0.01156266654531161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,31,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,63,0.012432000289360682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,8191,0.5477973222732544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,4095,0.28749332825342816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,255,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,255,0.012485332787036896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,511,0.01190399999419848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,511,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,511,0.01251199965675672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1023,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1023,0.013536000003417334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,8191,0.5598506530125936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1023,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,2047,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,2047,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,2047,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,4095,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,4095,0.025642665723959606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,4095,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,4095,0.02380266785621643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,8191,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,8191,0.04156800111134847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,8191,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,8191,0.03621866554021835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1023,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1,0.044064000248909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1,0.03995199998219808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,2047,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,3,0.045909335215886436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,15,0.05585599939028422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,15,0.04981866478919983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,31,0.056330665946006775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,31,0.0499946673711141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,63,0.05615466833114624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,63,0.05003199974695841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,127,0.06605333089828491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,3,0.0399893323580424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,127,0.060175999999046326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,7,0.047968000173568726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,255,0.09113066395123799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,511,0.1606773336728414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,7,0.03972800076007843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1023,0.28803733984629315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1023,0.2746293346087138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,2047,0.5378613471984863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,2047,0.5193599859873453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,255,0.09732266267140706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,4095,1.0044533411661785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,511,0.15250666936238608
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1,0.03772266705830892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,3,0.04568000137805939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,3,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,7,0.04633066554864248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,7,0.039706667264302574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,15,0.05605333546797434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,15,0.04987733562787374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,31,0.05625066657861074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,31,0.05028266708056132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,63,0.055829331278800964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,63,0.0524533341328303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,127,0.06635733445485432
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,127,0.060133333007494606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,255,0.09719467163085938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,255,0.09105599919954936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1,0.0440533310174942
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,511,0.1527253290017446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1023,0.28565865755081177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1023,0.27510933081309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,2047,0.5394186576207479
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,2047,0.5192746718724569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,4095,1.0539946556091309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,4095,1.0067306359608967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1,0.04385599990685781
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1,0.037791999677817024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,3,0.04587733248869578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,3,0.0396373321612676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,7,0.045978665351867676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,7,0.041482667128245033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,15,0.05605333546797434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,15,0.05003199974695841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,31,0.056234667698542275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,31,0.05014933149019877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,63,0.05597866574923197
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,63,0.049770668148994446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,127,0.0663679987192154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,127,0.06025066475073496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,255,0.09717866778373718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,255,0.09109866619110107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,511,0.16063466668128967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,511,0.15263467033704123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1023,0.28680533170700073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,2047,0.54366401831309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,2047,0.5206559896469116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,4095,1.0437813599904378
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,511,0.15999466180801392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,4095,1.0696746508280437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1023,0.27482134103775024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,4095,0.9879626433054606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,3,0.013354666531085968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,7,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,7,0.014655999839305878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,15,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,15,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,31,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,31,0.014453332871198654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,3,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,63,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,127,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,127,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,255,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,255,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,511,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1023,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,2047,0.03957866628964742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,2047,0.027077332139015198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,4095,0.05964800218741099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,4095,0.041893333196640015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1,0.009253333633144697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,3,0.00916800027092298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,7,0.00997866690158844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,15,0.009301333377758661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,31,0.010245333115259806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,63,0.010506667196750641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,63,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1023,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,255,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,255,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,511,0.026736001173655193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1023,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1023,0.03160533308982849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,2047,0.039749334255854286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,2047,0.040133332212766014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,4095,0.06225599845250448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,4095,0.05993066728115082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,8191,0.10769599676132202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,8191,0.09532800316810608
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,16383,0.19538132349650064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,16383,0.16891199350357056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,511,0.023503998915354412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,3,0.00902399979531765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,7,0.009226666763424873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,7,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,15,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,63,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,127,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,255,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,511,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,511,0.027295999228954315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1023,0.02945599953333537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1023,0.031397332747777305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,2047,0.041573333243529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,2047,0.041135999063650765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,4095,0.06406400104363759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,4095,0.060319999853769936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,8191,0.10749333103497823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,8191,0.09708266456921895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,16383,0.1958720088005066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,16383,0.16929600636164346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,3,0.009253333633144697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1,0.008885333314538002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,7,0.008976000050703684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,15,0.00879466657837232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1,0.009429333110650381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,127,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,511,0.027141332626342773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1023,0.029306667546431225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1023,0.031311998764673867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,2047,0.04052799940109253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,2047,0.04162133236726125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,63,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,4095,0.0626986672480901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,4095,0.060122668743133545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,8191,0.10752532879511516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,255,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,16383,0.19572800397872925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,16383,0.16893333196640015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,63,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,511,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1023,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,2047,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,2047,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,4095,0.016224000602960587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,4095,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,8191,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,8191,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,16383,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,16383,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1,0.009173333023985228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,3,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,7,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,7,0.008933333059151968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,15,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,31,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,31,0.012106666962305704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,63,0.015322666615247726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,63,0.012624000509579977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,8191,0.09660800298055013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,255,0.016778666526079178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,255,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,511,0.022917332748572033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,511,0.0207893339296182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1023,0.03252800057331721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1023,0.029696000119050343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,2047,0.05194666484991709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,2047,0.04773333172003428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,4095,0.090938667456309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,4095,0.08653333783149719
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,8191,0.17099199692408243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,8191,0.16268799702326456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,16383,0.32869867483774823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,16383,0.31356267134348553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,127,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,7,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,15,0.014709333578745524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,31,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,31,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,63,0.015487999965747198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,127,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,127,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,255,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,255,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,511,0.02250133454799652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,511,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1023,0.03328000009059906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1023,0.029370665550231934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,2047,0.05234666665395101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,2047,0.048165331284205117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,4095,0.09145599603652954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,4095,0.08679466446240743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,8191,0.17125866810480753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,8191,0.16273066401481628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,16383,0.32920000950495404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,16383,0.31328000624974567
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1,0.009050666665037474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,3,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,7,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,15,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,31,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,31,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,63,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,63,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,127,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,255,0.01754666616519292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,255,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,511,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,511,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1023,0.03359466542800268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1023,0.029306667546431225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,2047,0.05346666773160299
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,2047,0.048394665122032166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,4095,0.09298666318257649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,8191,0.17101333538691202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,8191,0.16078933080037436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,16383,0.32862399021784466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,3,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,4095,0.08679999907811482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,31,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,16383,0.3147146701812744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,255,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,511,0.012015999605258306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1023,0.013637332866589228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,2047,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,2047,0.01623999948302905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,4095,0.018277333428462345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,255,0.010015999898314476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,4095,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,8191,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,8191,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,16383,0.02346666653951009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,16383,0.022485333184401195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1,0.08326933284600575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1,0.07051200171311696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,3,0.08629332979520161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,3,0.07288533449172974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,7,0.08679466446240743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,7,0.07497066756089528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,63,0.011653333902359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,15,0.0935040016969045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,31,0.10548266768455505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,31,0.09504000345865886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,63,0.1053600013256073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,63,0.0949173370997111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,127,0.1239359974861145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,127,0.11386666695276897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,255,0.1750613252321879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,511,0.31172800064086914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,511,0.2977546652158101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1023,0.5634826819101969
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1023,0.5373813311258951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,2047,1.1197866598765056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,255,0.18730133771896362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1,0.0703413337469101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,3,0.08514666557312012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,3,0.07256533205509186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,15,0.10531733433405559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,7,0.08685333530108134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,7,0.07655466596285503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,15,0.10546666383743286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,15,0.09493333101272583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,31,0.1055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,31,0.09507733583450317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,63,0.10541333754857381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,63,0.0952959954738617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1,0.08267733454704285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,127,0.11531733473141988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,255,0.18742400407791138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,255,0.17728533347447714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,511,0.3108373284339905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,511,0.2914186716079712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1023,0.5633813142776489
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1023,0.5359573364257812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,2047,1.0211679935455322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,127,0.12389866511027019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,2047,1.018341302871704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1,0.07056533296902974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,3,0.08628267049789429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,3,0.07238933444023132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,7,0.08719467123349507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,7,0.07609599828720093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,15,0.10532266894976298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,15,0.09321066737174988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,31,0.1050879955291748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,31,0.09461333354314168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,63,0.10518399874369304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,63,0.0950933297475179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,127,0.1241439978281657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,127,0.11428800225257874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,255,0.18712000052134195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,255,0.17504000663757324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,511,0.31034133831659955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,511,0.2979946732521057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1023,0.5688106616338094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1023,0.5361493428548177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,2047,1.0733919938405354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,2047,1.0850773652394612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1,0.018842666099468868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,3,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,3,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,7,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,7,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,15,0.01874133323629697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,15,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,31,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,31,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,2047,1.0368106365203857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,63,0.016832000265518825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,127,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,255,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,255,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,511,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1,0.08342933654785156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,511,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1023,0.03766933331886927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,2047,0.06186666587988535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,63,0.018602666755517323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,2047,0.04331199824810028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,3,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,3,0.01565333331624667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,7,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,7,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,15,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,15,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,31,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,31,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,63,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,63,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,127,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,127,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,255,0.020954666038354237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,255,0.023178666830062866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,511,0.02992533395687739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,511,0.033013333876927696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1023,0.05023466547330221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1023,0.05209066470464071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,2047,0.0906880001227061
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,127,0.01684800038735072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,2047,0.08844799796740214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,4095,0.17113600174585977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,4095,0.16237866878509521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,8191,0.3314879933993022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,8191,0.3083573381106059
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1023,0.025600001215934753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,16383,0.6602026621500651
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,3,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,3,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,7,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,7,0.015498666713635126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,15,0.014485333114862442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,15,0.016773333152135212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,31,0.013728000223636627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,31,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,63,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,63,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,127,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,127,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,255,0.02094399929046631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,255,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,511,0.03128000100453695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,511,0.033615998923778534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1023,0.05188799897829691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1023,0.05211733281612396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,16383,0.6018133163452148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,2047,0.09113599856694539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,2047,0.08898133039474487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,4095,0.17283199230829874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,4095,0.16250133514404297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,8191,0.3381386597951253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,8191,0.3095253308614095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,16383,0.6687359809875488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,3,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,7,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,7,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,15,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,15,0.016261332978804905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,31,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,31,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,63,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,127,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,127,0.019167999426523846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1,0.015861333658297855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,255,0.023039999107519787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,511,0.03145600110292435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,511,0.03324799984693527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1023,0.050885334610939026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,16383,0.6027733484903971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1023,0.051738664507865906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,63,0.01904533306757609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,2047,0.09076799949010213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,2047,0.08859200278917949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,4095,0.17106133699417114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,4095,0.16266666849454245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,8191,0.3349440097808838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,8191,0.30851733684539795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,16383,0.6764266490936279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,3,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,63,0.01181866725285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,255,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,511,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1023,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,16383,0.6032586495081583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,2047,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,2047,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,4095,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,8191,0.022304000953833263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,8191,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,16383,0.03941866755485535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,16383,0.027376001079877216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1,0.1337440013885498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,3,0.1648906668027242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,3,0.1386133333047231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,7,0.1670560042063395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,7,0.1444906691710154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,15,0.2039946715037028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1,0.1609226663907369
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,15,0.18155733744303384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,31,0.20353599389394125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,31,0.18130133549372354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,63,0.2039946715037028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,63,0.1829493244489034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,127,0.23468265930811563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,127,0.22020800908406576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,255,0.3654559850692749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,255,0.3428586721420288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,511,0.6116533279418945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,511,0.5844533443450928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1023,1.1150773366292317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1023,1.0593012968699138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,3,0.16473600268363953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,3,0.1381760040918986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,7,0.16716265678405762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,7,0.14433067043622336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,15,0.20456000169118246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,15,0.18099733193715414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,31,0.20367467403411865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,31,0.1811413367589315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,63,0.20389332373936972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1,0.13205333550771078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,127,0.2206666668256124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,255,0.36556267738342285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,255,0.34330666065216064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1,0.1588373382886251
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,511,0.61354132493337
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,127,0.2423093318939209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,511,0.585477352142334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1,0.15890666842460632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1023,1.1263946692148845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1023,1.0612693627675374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,3,0.16472533345222473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,3,0.13792533675829569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,7,0.16671999295552573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,7,0.1442346672217051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,63,0.18170666694641113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,15,0.18097599347432455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,31,0.20415467023849487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,31,0.1812373399734497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,63,0.20576000213623047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,63,0.18129066626230875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,127,0.2423893411954244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,127,0.22202666600545248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,255,0.3657493193944295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,255,0.34324800968170166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,15,0.20382400353749594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,511,0.5847520033518473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1023,1.1357173124949138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1023,1.0888426303863525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1,0.13404267032941183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,3,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,3,0.021856000026067097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,7,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,7,0.02183466653029124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,15,0.025626666843891144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,15,0.023024000227451324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,31,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,31,0.022997332115968067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,63,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,63,0.02180800090233485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,127,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,127,0.021701333423455555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,255,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,511,0.03825066735347112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,511,0.02762666592995326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1023,0.05866666634877523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1023,0.04038933416207632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,1,0.31091733773549396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,1,0.2584639986356099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,3,0.322378675142924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,255,0.02332799881696701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,7,0.32628800471623737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,7,0.2823253273963928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,15,0.4002559979756673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,15,0.3535999854405721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,31,0.40037333965301514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,31,0.35556801160176593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,63,0.40249065558115643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,63,0.35738134384155273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,127,0.4763146638870239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,127,0.4342720111211141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,511,0.6270720163981119
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,255,0.7225706577301025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,255,0.6781760056813558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,1,0.3102506597836812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,1,0.25843199094136554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,3,0.3237920006116231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,7,0.3286400039990743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,7,0.283951997756958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,15,0.4004319906234741
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,15,0.3550293445587158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,31,0.40043731530507404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,31,0.3557013273239136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,63,0.4021759827931722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,3,0.2694399952888489
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,127,0.476149320602417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,127,0.43481067816416424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,255,0.7238346735636393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,255,0.6773386796315511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,1,0.3102506597836812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,1,0.25869866212209064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,3,0.3227573235829671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,3,0.269381324450175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,7,0.3286400039990743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,7,0.28351465861002606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,15,0.40035200119018555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,15,0.353333314259847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,31,0.4004799922307332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,31,0.3555786609649658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,63,0.4029013315836589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,127,0.4782239993413289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,127,0.4349493185679118
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,255,0.7477973302205404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,3,0.26930665969848633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,1,0.0396373321612676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,255,0.6781973044077555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,1,0.03382933388153712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,3,0.039749334255854286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,3,0.033759998778502144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,7,0.04009599983692169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,7,0.033546666304270424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,15,0.039834665755430855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,15,0.03369600077470144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,31,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,31,0.03506666670242945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,63,0.03974399964014689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,63,0.03526933242877325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,127,0.039808000127474465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,127,0.035349334279696144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,255,0.04195733368396759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,255,0.03541333228349686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,1,0.6142986615498861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,1,0.5069546699523926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,63,0.35946134726206463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,3,0.637386679649353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,7,0.6468799908955892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,7,0.5563466548919678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,15,0.7950666745503744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,63,0.3575626611709595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,31,0.7933706442515055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,31,0.7040906747182211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,63,0.7978400389353434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,63,0.7309652964274088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,15,0.70033065478007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,127,0.9453653494517008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,3,0.5297066768010458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,1,0.6127359867095947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,1,0.5089759826660156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,3,0.6381920178731283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,3,0.5310293436050415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,7,0.6521600087483724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,7,0.557861328125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,127,0.8600160280863444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,15,0.7956746419270834
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,15,0.7008852958679199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,31,0.7732319831848145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,63,0.7992853323618571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,63,0.7057973543802897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,127,0.9494400024414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,127,0.8597066402435303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,31,0.7048266728719076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,1,0.6112639904022217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,1,0.5109013319015503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,3,0.6391253471374512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,3,0.5320106744766235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,7,0.5579839944839478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,15,0.7935787041982015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,31,0.7931146621704102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,31,0.7048959732055664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,7,0.6526293357213339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,63,0.7071146965026855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,15,0.6996426582336426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,127,0.9886826674143473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,1,0.06834666430950165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,1,0.056277334690093994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,3,0.06791999936103821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,3,0.05583466589450836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,63,0.8000053564707438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,7,0.06822933256626129
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,7,0.05624533196290334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,15,0.06854400038719177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,15,0.056373332937558494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,31,0.06815466781457265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,31,0.05640000104904175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,63,0.06832533578077953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,127,0.8648800055185953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,127,0.058176000912984215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,3,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,7,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,7,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,15,0.019461333751678467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,15,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,31,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,31,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,63,0.05585599939028422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,127,0.06933866441249847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,127,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,255,0.02958933264017105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,255,0.029743999242782593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,511,0.04794133206208547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,511,0.04619200030962626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1023,0.08105599880218506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1023,0.0806879997253418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,63,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,63,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,2047,0.14620799819628397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,127,0.021717332303524017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,4095,0.2836373249689738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,4095,0.2793546716372172
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,8191,0.5538346767425537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,8191,0.5458773374557495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,16383,1.1605599721272786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,3,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,16383,1.310421307881673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,7,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,15,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,15,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,31,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,31,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,63,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,63,0.017504000415404636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,127,0.021359999974568684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,3,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,255,0.02959999938805898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,7,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,511,0.04738666613896688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,511,0.04560000201066335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1023,0.08071466783682506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1023,0.07855466504891713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,2047,0.14844800035158792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,2047,0.1462453305721283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,127,0.02077866718173027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,2047,0.14867732922236124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,8191,0.5627306699752808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,8191,0.5469280083974203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,4095,0.2845813234647115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,255,0.029232000311215717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,16383,1.2531306743621826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,3,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,3,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,7,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,7,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,15,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,15,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,31,0.01904533306757609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,31,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,63,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,63,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,127,0.02146133283774058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,4095,0.2794559995333354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,255,0.029946667452653248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,255,0.02956799914439519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,511,0.047914668917655945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,511,0.04586666822433472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1023,0.08110933502515157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1023,0.07880533238252004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,2047,0.14889066418011984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,2047,0.1464906632900238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,127,0.01977066695690155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,4095,0.2845120032628377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,4095,0.2806346615155538
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,16383,1.309930642445882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,8191,0.5651946862538656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,8191,0.5480853319168091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,3,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,16383,1.3680267333984375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,16383,1.446608066558838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,15,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,31,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,511,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1023,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1023,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,2047,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,2047,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,4095,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,4095,0.019744000087181728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,8191,0.03809600075085958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,8191,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,16383,0.05922133227189382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,16383,0.04190933207670847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,1,1.2212906678517659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,7,0.011648000528415045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,3,1.2705492973327637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,3,1.0561760266621907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,1,1.0070079962412517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,7,1.1093546549479167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,15,1.585759957631429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,15,1.3586452802022297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,7,1.2913013299306233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,31,1.5869332949320476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,31,1.404431978861491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,63,1.6055466334025066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,63,1.409455935160319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,1,1.2195093631744385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,1,1.0152160326639812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,3,1.2715413570404053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,3,1.058293342590332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,7,1.3050133387247722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,7,1.116101344426473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,15,1.5878292719523113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,31,1.6024319330851238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,31,1.41376527150472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,63,1.6372052828470867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,63,1.4323093096415203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,15,1.398896058400472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,1,1.2226826349894206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,1,1.016266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,3,1.2791732947031658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,7,1.3234933217366536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,7,1.1158506870269775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,15,1.607850710550944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,15,1.4086613655090332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,3,1.0622506936391194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,31,1.6307627360026042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,31,1.4302453994750977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,1,0.1252959966659546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,63,1.653146743774414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,63,1.472090721130371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,1,0.10517866412798564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,3,0.12568533420562744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,3,0.10544533530871074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,7,0.12483732899030049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,7,0.10528533657391866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,15,0.12435199817021687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,15,0.10422399640083313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,31,0.12450133760770161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,31,0.10514666636784871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,63,0.1253439982732137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,63,0.1051093339920044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,1,2.1778720219930015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,1,2.611839930216471
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,3,2.6888214747111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,3,2.222640037536621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,7,2.73252805074056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,7,2.3089547157287598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,15,3.2385759353637695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,15,2.863114674886068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,31,3.2387574513753257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,31,2.8616053263346353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,1,2.6403199831644693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,3,2.7197707494099936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,1,2.208240032196045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,3,2.242570718129476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,7,2.760085423787435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,7,2.338085333506266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,15,2.877317428588867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,15,3.2531468073527017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,31,2.886538823445638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,1,2.245898723602295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,1,2.696234703063965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,3,2.7582880655924478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,31,3.2524000803629556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,3,2.2851573626200357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,7,2.7960427602132163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,7,2.3684746424357095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,15,3.266362508138021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,31,3.2674506505330405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,31,2.9045279820760093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,1,0.23620800177256265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,1,0.19509865840276083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,3,0.23459200064341226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,3,0.19579732418060303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,7,0.23664534091949463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,15,0.23642667134602866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,15,2.889946619669596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,15,0.1957706610361735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1,0.02749866743882497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,31,0.23753066857655844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,3,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,7,0.1959786613782247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,31,0.19583467642466226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,7,0.023658665517965954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,15,0.031712000568707786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,15,0.02914133419593175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,31,0.03133333226044973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,31,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,63,0.031471999982992806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,63,0.02979733298222224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,127,0.037045332292715706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,127,0.033786666889985405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,255,0.0545066644748052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,255,0.04978133241335551
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,511,0.08674133817354839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,511,0.08296533425649007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1023,0.15252799789110819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1023,0.1479466656843821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,3,0.023247999449570973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,7,0.02754133443037669
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,4095,0.5492159922917684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,4095,0.531760017077128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,8191,1.0790026982625325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,8191,1.0477866331736247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1,0.025605333348115284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1,0.02348800003528595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,3,0.027493332823117573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,3,0.023311999936898548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,7,0.027114666998386383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,7,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,15,0.0315786674618721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,15,0.029637334247430164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,31,0.03125333289305369
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,31,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,63,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,63,0.02916266769170761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,127,0.035802667339642845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,127,0.03350399931271871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,255,0.05288533369700114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,255,0.050026665131251015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,511,0.08685333530108134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,511,0.082997332016627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1023,0.15242133537928262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1023,0.14673067132631937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,2047,0.283733328183492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,2047,0.2755626638730367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,2047,0.2757013241449992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,4095,0.5496533314387003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,4095,0.5312960147857666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1,0.025797332326571148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1,0.02351466566324234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,3,0.027210667729377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,8191,1.0487360159556072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,3,0.02364266663789749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,7,0.027098665634791057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,7,0.023711999257405598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,15,0.03136533250411352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,15,0.02922666569550832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,31,0.031221332649389904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,31,0.029616000751654308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,63,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,63,0.02921066681543986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,127,0.037920000652472176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,127,0.03376533339420954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,8191,1.0962613423665364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,255,0.05022400120894114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,511,0.0869760016600291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,511,0.08296533425649007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1023,0.1525226632754008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1023,0.14623467127482095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,2047,0.2834879954655965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,255,0.05386666456858317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,4095,0.5519359906514486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,2047,0.28412266572316486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,8191,1.110368013381958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,2047,0.2754559914271037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,3,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,7,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,4095,0.5335893233617147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,63,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,127,0.01267733300725619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,255,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,255,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,511,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,511,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1023,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1023,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,2047,0.02102400114138921
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,2047,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,4095,0.037503999968369804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,4095,0.02514133354028066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,31,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,8191,0.04182933270931244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1,0.03551466763019562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1,0.03136000037193298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1,0.031717332700888314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,8191,1.0554347038269043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,3,0.0359199990828832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,3,0.031541332602500916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,3,0.031871999303499855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,8191,0.05758399764696757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1,0.03568000098069509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,7,0.03330666571855545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,7,0.0329120010137558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,15,0.03786666691303253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,3,0.035930665830771126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,15,0.03323733309904734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,15,0.03321066747109095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,31,0.04354666670163473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,7,0.03793599953254064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,7,0.03779733429352442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,31,0.03996799886226654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,63,0.0439573327700297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,63,0.04446933170159658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,15,0.038032000263532005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,63,0.039706667264302574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,127,0.04585599899291992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,127,0.04601066807905833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,31,0.04391466577847799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,31,0.03996799886226654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,255,0.04577599962552389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,255,0.053216000398000084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,255,0.041519999504089355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,63,0.03988266736268997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,511,0.04569066564242045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,511,0.07660266757011414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,511,0.039877332746982574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,127,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,127,0.04015466570854187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1023,0.12570133805274963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1023,0.03982933362325033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1023,0.12142933408419292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,2047,0.045653333266576133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,2047,0.22407466173171997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,2047,0.03977066775163015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1023,0.04594666759173075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,2047,0.227183997631073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,4095,0.04570133487383524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,4095,0.42282132307688397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,255,0.04830400149027506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1,0.03579200059175491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1,0.035674666364987694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,1,0.03148799886306127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,511,0.07238933444023132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,3,0.036015999813874565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,3,0.035599999129772186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,3,0.031290667752424874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,4095,0.040991999208927155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,7,0.03737599899371465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,7,0.03779733429352442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,7,0.031770666440327965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1,0.03145066648721695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,7,0.031845333675543465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,15,0.03772266705830892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,15,0.037808001041412354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,3,0.03126399964094162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,15,0.033413333197434746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,4095,0.41464531421661377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,31,0.04390933116277059
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,31,0.039877332746982574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,31,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,63,0.044405331214269005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,63,0.04419733087221781
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,63,0.03988266736268997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,15,0.03342933456103007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,63,0.03985599925120672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,127,0.045941332976023354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,127,0.045941332976023354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,127,0.03976000100374222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,127,0.03984000037113825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,255,0.045797333121299744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,255,0.05201066533724467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,255,0.04009066770474116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,511,0.04603200157483419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,511,0.07694933315118153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,511,0.0397119993964831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,511,0.07234666744867961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1023,0.04565866788228353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1023,0.12593600153923035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1023,0.1218933363755544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,2047,0.04590400060017904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,2047,0.22433066368103027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,2047,0.04048533240954081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,255,0.04898133377234141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,2047,0.22011733055114746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,4095,0.04660266637802124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,1023,0.039850667119026184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,4095,0.41522665818532306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,3,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,3,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,4095,0.4261866807937622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,3,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,3,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,7,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,7,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,7,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,7,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,15,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,15,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,15,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,31,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,31,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,31,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,4095,0.04200000067551931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,31,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,63,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,31,0.045269335309664406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,63,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,127,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,127,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,127,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,255,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,255,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,63,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,63,0.013514666507641474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,511,0.013951999445756277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,15,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,511,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1023,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1023,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,2047,0.01599466676513354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,2047,0.025706666211287182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,2047,0.015658666690190632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,511,0.015381333728631338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,4095,0.01595199977358182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,4095,0.04342400034268697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,4095,0.01590399940808614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,4095,0.03544000039498011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1,0.009306666751702627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,3,0.009056000038981438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,3,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,7,0.009258666386206945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,7,0.009130666653315226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1,0.009039999917149544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,15,0.00897066667675972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,15,0.008816000074148178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,2047,0.02518933266401291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,31,0.009125333279371262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,31,0.00902399979531765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,63,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,127,0.009797333429257074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,127,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,127,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,127,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,255,0.009808000177145004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,255,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,511,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,511,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,511,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1023,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1023,0.02367466688156128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1023,0.023562667270501454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1023,0.02741333345572154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,2047,0.018922666708628338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,2047,0.029722665747006733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,2047,0.02313599983851115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,4095,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,4095,0.04022933294375738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,4095,0.023567999402681988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,4095,0.040192000567913055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,8191,0.06216000020503998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,8191,0.023823998868465424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,8191,0.05982933441797892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,16383,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,16383,0.10587732990582784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,16383,0.023391999304294586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,16383,0.09718933701515198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1,0.008789333204428354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,2047,0.031194667021433514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,3,0.009445333232482275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,8191,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,3,0.009306666751702627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,7,0.009285333255926767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,7,0.009039999917149544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,7,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,15,0.009056000038981438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1,0.009082666908701261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,31,0.009056000038981438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,127,0.009674666449427605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,127,0.010474666953086853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,127,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,127,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,255,0.010512000570694605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,511,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,511,0.020794666061798733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,511,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,511,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1023,0.01870399961868922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1023,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,1023,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1023,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,2047,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,2047,0.02940800040960312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,15,0.009183999771873156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,2047,0.023045333723227184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,2047,0.031701333820819855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,4095,0.019173332800467808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,4095,0.039834665755430855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,4095,0.02311466634273529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,4095,0.04031999905904134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,8191,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,8191,0.062224000692367554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,8191,0.023152001202106476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,8191,0.0601440022389094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,16383,0.01889066646496455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,16383,0.10693867007891338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,16383,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,31,0.009077333534757296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,3,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,7,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,63,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,127,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,255,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,16383,0.09713066617647807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,511,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1023,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1023,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1023,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,2047,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,2047,0.012389333297808966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,2047,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,511,0.011898666620254517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,4095,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,63,0.011610666910807291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,4095,0.011994666109482447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,4095,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,8191,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,8191,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,8191,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,2047,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,16383,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,16383,0.017701332767804463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,4095,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,16383,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,3,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,3,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,3,0.016330666840076447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,3,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,7,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,7,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,7,0.016352000335852306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,7,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,15,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,15,0.016143999993801117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,15,0.014655999839305878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,31,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,31,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,31,0.015530666957298914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,31,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,63,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,63,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,63,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,127,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,127,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,127,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,127,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,8191,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,255,0.01540800059835116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,255,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,255,0.020954666038354237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,511,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,511,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,16383,0.019850666324297588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,511,0.021189334491888683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1023,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1023,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1023,0.021317332983016968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1023,0.023423999547958374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,2047,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,2047,0.031514666974544525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,2047,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,2047,0.033573334415753685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,63,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,4095,0.015504000087579092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,4095,0.052149335543314614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,4095,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,4095,0.05239466826121012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,8191,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,8191,0.09284266829490662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,255,0.02091199904680252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,8191,0.020949333906173706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,8191,0.08871466914812724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,16383,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,16383,0.17098132769266763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,16383,0.02146133283774058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,1,0.015413332730531693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,3,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,3,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,3,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,3,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,511,0.021007999777793884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,7,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,16383,0.16013333201408386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,7,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,15,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,15,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,15,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,31,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,31,0.013487999637921652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,31,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,31,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,7,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,63,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,63,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,63,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,7,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,127,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,127,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,127,0.021354667842388153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,15,0.014042666802803675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,255,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,255,0.015882667154073715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,255,0.021498667697111767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,511,0.015978666643301647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,511,0.017701332767804463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,511,0.021888000269730885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,511,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1023,0.015482666591803232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1023,0.02333866556485494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,127,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1023,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,2047,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,2047,0.033488000432650246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,2047,0.021375998854637146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,2047,0.03369600077470144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,4095,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,4095,0.05219733218352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,4095,0.021194666624069214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,1023,0.021898667017618816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,255,0.021018666525681812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,8191,0.09108266234397888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,8191,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,8191,0.08730133374532063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,16383,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,63,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,16383,0.17113600174585977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,16383,0.020949333906173706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,4095,0.051813334226608276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,3,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,3,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,7,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,15,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,15,0.012373333175977072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,16383,0.15967466433842978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,127,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,255,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,511,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,511,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1023,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1023,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,2047,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,2047,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,2047,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,4095,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,4095,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,4095,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,4095,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,8191,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,8191,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,8191,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,8191,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,16383,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,16383,0.02086399992307027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,16383,0.012485332787036896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,16383,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1,0.06632533172766368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1,0.06613333523273468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1,0.05435200035572052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1,0.05388799806435903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,3,0.06663999954859416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,3,0.06665066878000896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,3,0.05398933092753092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,3,0.05420800050099691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,7,0.06841066479682922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,7,0.06826133529345195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,7,0.056143999099731445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,15,0.0683786670366923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,15,0.06886399785677592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,15,0.058287998040517174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,15,0.0581226646900177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,31,0.08193066716194153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,31,0.08283733328183492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,31,0.07252266506354015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,31,0.07276799778143565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,63,0.0827946662902832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,63,0.08285866677761078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,63,0.07243200143178304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,63,0.07267199953397115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,127,0.08272000153859456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,127,0.08270933230717976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,127,0.07257600128650665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,127,0.07458133498827617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,255,0.0830080012480418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,255,0.09724799791971843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,255,0.07253866891066234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,255,0.08819199601809184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,511,0.08284266789754231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,511,0.14442132910092673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,511,0.07262399792671204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,511,0.13205867012341818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1023,0.08292800188064575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1023,0.23844265937805176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1023,0.22859734296798706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,7,0.05658666789531708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,2047,0.08309866487979889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,2047,0.07299200197060902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,2047,0.41886401176452637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1,0.06654933094978333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1,0.06596800188223521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,1,0.054485330979029335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1,0.05426666637261709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,8191,0.015583999454975128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,3,0.0666186660528183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,3,0.05452266832192739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,3,0.054325332244237266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,7,0.06870399912198384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1023,0.07233066856861115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,7,0.05594133337338766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,7,0.05611200133959452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,3,0.06604800124963124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,15,0.06862933437029521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,15,0.06871999800205231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,15,0.05789333085219065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,15,0.058389330903689064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,31,0.08092799782752991
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,31,0.08265600105126698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,31,0.07257600128650665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,31,0.07297599812348683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,63,0.08295999964078267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,63,0.08285866677761078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,63,0.07281599938869476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,63,0.07289066910743713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,127,0.0828000009059906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,127,0.08455999692281087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,127,0.07316266496976216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,127,0.07452266911665599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,2047,0.430618683497111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,255,0.09940800070762634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,255,0.07309866448243459
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,255,0.08854400118192036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,511,0.0830506682395935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,511,0.14428800344467163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,7,0.06850133339564006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,511,0.13639466961224875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1023,0.08342933654785156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,255,0.08284266789754231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,1023,0.073594664533933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1023,0.22914665937423706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,2047,0.08310933411121368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,2047,0.4331786632537842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,2047,0.07258133093516032
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,2047,0.4191519816716512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,511,0.07258666555086772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,3,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,3,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,3,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,7,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,7,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,7,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,7,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1,0.01580799991885821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,15,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,15,0.01590399940808614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,15,0.015450666348139444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,31,0.015557333827018738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,31,0.015493333339691162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,31,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,63,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,63,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,15,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,63,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,127,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,127,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,127,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,127,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,255,0.016693333784739178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,255,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,255,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,255,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,511,0.016864000509182613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,511,0.018960000326236088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,31,0.015637333194414776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,511,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1023,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1023,0.02571733295917511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1023,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1023,0.02349333216746648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,2047,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,511,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,2047,0.04322666426499685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,2047,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,2047,0.03545066714286804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,63,0.014837333311637243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,3,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,3,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,3,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,7,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,7,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,7,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,15,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,15,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,15,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,15,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,31,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,31,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,31,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,31,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,63,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,63,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,63,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,63,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,127,0.021295999487241108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,127,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,127,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,127,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,255,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,255,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,7,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,255,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,511,0.02346666653951009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,511,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1023,0.021317332983016968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1023,0.03194133440653483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1023,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1023,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,2047,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,255,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,2047,0.04808533191680908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,2047,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,2047,0.03990400085846583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,4095,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,4095,0.07692266503969829
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,4095,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,4095,0.06844800213972728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,8191,0.02109866589307785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,8191,0.13598400354385376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,8191,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,8191,0.12492799758911133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,511,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,16383,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,511,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,16383,0.25333333015441895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,16383,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1,0.01740266631046931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,1,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,3,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,3,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,3,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,3,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,7,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,7,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,7,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,7,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,15,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,15,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,15,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,15,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,31,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,31,0.01534933348496755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,31,0.015557333827018738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,63,0.02165866643190384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,63,0.0176959993938605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,63,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,127,0.021930667261282604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,127,0.021903999149799347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,127,0.017583999782800674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,31,0.019440000255902607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,255,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,16383,0.23837333917617798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,255,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,255,0.016805333395799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,511,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,511,0.023525332411130268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,511,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,511,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1023,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,127,0.017717332889636356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,1023,0.01758933315674464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,63,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1023,0.027818667391935985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,2047,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,2047,0.04825599988301595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,2047,0.04044266790151596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,4095,0.021615999440352123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,4095,0.07685866455237071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,4095,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1023,0.03145600110292435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,8191,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,8191,0.13588800032933554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,2047,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,8191,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,8191,0.12458667159080505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,16383,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,16383,0.2531680067380269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,16383,0.23802133401234946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,4095,0.06869333485762279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,63,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,16383,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,127,0.012453333785136541
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,255,0.011717333147923151
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,255,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,511,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1023,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,255,0.02109866589307785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,2047,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,2047,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,4095,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,4095,0.01578666642308235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,4095,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1023,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,4095,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,8191,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,8191,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,8191,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,8191,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,16383,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,16383,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,16383,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,16383,0.023317334552605946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1,0.12405866384506226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1,0.12377599875132243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1,0.10165866216023763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,3,0.12381333112716675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,3,0.12451199690500896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,3,0.10148266951243083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,3,0.10108266274134318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,2047,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,7,0.12804266810417175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,7,0.10540800293286641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,15,0.12989333271980286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,15,0.13013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,7,0.12787733475367227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,15,0.10958932836850484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,15,0.10949866970380147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,31,0.15681599577267966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,31,0.15664000312487283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,31,0.138373335202535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,31,0.137882669766744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,63,0.15850133697191873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,63,0.15863466262817383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,63,0.14008000493049622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,63,0.1383519967397054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,127,0.1604266663392385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,127,0.14034133156140646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,127,0.14033599694569907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,255,0.1607093314329783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1,0.10142933328946431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,255,0.14012266198794046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,127,0.1588213344415029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,511,0.16035200158754984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,511,0.2797120014826457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,7,0.10552000006039937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,511,0.13874133427937826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,255,0.18732800086339316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1023,0.16061866283416748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1023,0.1400053302447001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,255,0.16883200407028198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1023,0.44753066698710126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1,0.12150399883588155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,1,0.10150399804115295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1,0.10132799545923869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,511,0.26157333453496295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,3,0.12388799587885539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,3,0.12379733721415202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,3,0.10115733742713928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,7,0.12799466649691263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,7,0.12800533572832742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,7,0.10493866602579753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,7,0.1050933301448822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,3,0.10145066181818645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,15,0.12987732887268066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,15,0.13009066383043924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,15,0.10950932900110881
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,15,0.10939733187357585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1,0.12403733531634013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,31,0.1567093332608541
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,31,0.1381493310133616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,31,0.13803199927012125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,63,0.15848533312479654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,63,0.15864533185958862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,63,0.13834133744239807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,63,0.1383093297481537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,127,0.1597493290901184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1023,0.4675840139389038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,127,0.15874666968981424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,127,0.13858133554458618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,127,0.13978667060534158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,255,0.16060800353686014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,255,0.18756800889968872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,255,0.140255997578303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,255,0.1679146687189738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,511,0.1607146660486857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,511,0.2803200085957845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,511,0.2609813412030538
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1023,0.1609386702378591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1023,0.4780106544494629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,31,0.15666666626930237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,511,0.14010666807492575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1023,0.44762134552001953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1,0.020010666300853092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,3,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,3,0.02067199970285098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,3,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,3,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,7,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,7,0.020069333414236706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,7,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,7,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,15,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,15,0.018954666952292126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,15,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,15,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,31,0.019487999379634857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,31,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,31,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,31,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,63,0.020432000358899433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,63,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,63,0.01912533367673556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,127,0.019007999449968338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,127,0.02004266654451688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,127,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,127,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,255,0.021013334393501282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,255,0.02027733375628789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,255,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,255,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,511,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,511,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,1023,0.14029332995414734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,511,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,511,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1023,0.020975999534130096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1023,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1023,0.03342399994532267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,1,0.24051199356714884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,1,0.24108266830444336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,1,0.1934559941291809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,1,0.19372800985972086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,3,0.2404586672782898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,3,0.2405653397242228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,3,0.19322667519251505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,3,0.19363200664520264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,63,0.019882666567961376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,7,0.24901866912841797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,7,0.20273600021998087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,7,0.2031946579615275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,15,0.25385600328445435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,15,0.2548639973004659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,15,0.21060800552368164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,15,0.21082133054733276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1023,0.04009066770474116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,31,0.30646934111913043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,7,0.24808533986409506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,31,0.26711465915044147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,31,0.26763200759887695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,63,0.308079997698466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,63,0.31007466713587445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,63,0.2697226603825887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,127,0.3121546705563863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,127,0.31228800614674884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,127,0.27131199836730957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,127,0.2720906734466553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,31,0.3065119981765747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,255,0.3123306632041931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,255,0.36823999881744385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,255,0.2731253306070964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,255,0.3285813331604004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,1,0.24065599838892618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,1,0.24078933397928873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,1,0.19538132349650064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,1,0.1956160068511963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,3,0.24106667439142862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,3,0.24065599838892618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,3,0.19567465782165527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,7,0.24837867418924967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,7,0.24917866786321005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,7,0.20163200298945108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,7,0.2037280003229777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,15,0.2527466615041097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,63,0.2693013350168864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,15,0.20991466442743936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,15,0.21193599700927734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,3,0.1958400011062622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,31,0.3060906728108724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,31,0.3060160080591838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,31,0.2674986720085144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,31,0.2674986720085144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,63,0.31011199951171875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,63,0.31014400720596313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,63,0.26954134305318195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,127,0.3121066689491272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,127,0.31228800614674884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,127,0.2718240022659302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,127,0.2715359926223755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,255,0.3125386635462443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,63,0.269381324450175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,255,0.36779733498891193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,255,0.2713013291358948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,1,0.029450667401154835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,1,0.03091199944416682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,1,0.028042666614055634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,1,0.02849599967400233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,3,0.031248000760873158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,15,0.2537226676940918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,3,0.031701333820819855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,3,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,255,0.3286186655362447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,7,0.031290667752424874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,7,0.029487999776999157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,7,0.02914133419593175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,15,0.03126399964094162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,15,0.02985599885384242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,15,0.029189333319664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,15,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,31,0.031199999153614044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,31,0.029616000751654308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,31,0.02921066681543986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,31,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,63,0.029669334491093952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,63,0.0295413335164388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,63,0.02922133356332779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,63,0.029189333319664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,127,0.02975466599067052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,127,0.029829333225886028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,127,0.029157333076000214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,127,0.02959466725587845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,255,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,255,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,255,0.0301706666747729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,3,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,7,0.029887999097506206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,1,0.4737066825230916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,1,0.37885332107543945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,1,0.37828266620635986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,3,0.49297066529591876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,3,0.47299734751383465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,1,0.47341867287953693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,3,0.37778667608896893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,7,0.48897067705790204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,7,0.4882826805114746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,7,0.39738667011260986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,7,0.3954133192698161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,255,0.029711998999118805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,15,0.49876264731089276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,15,0.49852800369262695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,15,0.4150826533635457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,15,0.41278398036956787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,31,0.6050879955291748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,31,0.6079626480738322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,31,0.5410240093866984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,31,0.5274933179219564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,63,0.6136320034662882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,3,0.37809598445892334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,63,0.531770666440328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,63,0.5316640138626099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,127,0.6127306620279948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,127,0.6164586544036865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,127,0.5343039830525717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,127,0.5355146725972494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,1,0.4737066825230916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,1,0.47205865383148193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,63,0.6123626629511515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,1,0.3827626705169678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,1,0.3827626705169678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,3,0.48816001415252686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,3,0.47355198860168457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,3,0.3823253313700358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,7,0.49085867404937744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,7,0.49002134799957275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,7,0.3970400094985962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,7,0.3986560106277466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,15,0.49960533777872723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,15,0.4986506700515747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,15,0.4281333287556966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,15,0.4148373206456502
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,31,0.6052159865697225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,31,0.6066293319066366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,31,0.5284159978230795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,31,0.5276000102361044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,63,0.5967359940210978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,63,0.611786683400472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,63,0.5376799901326498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,3,0.39611200491587323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,127,0.6167893409729004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,127,0.6425653298695883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,127,0.5358133316040039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,1,0.04974933465321859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,1,0.049770668148994446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,1,0.047728002071380615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,1,0.04654400050640106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,3,0.050053333242734276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,63,0.5323413213094076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,3,0.05011733373006185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,3,0.04571199913819631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,3,0.045754666129748024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,7,0.0498879998922348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,127,0.5360586643218994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,7,0.049973333875338234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,7,0.045791998505592346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,7,0.046394666035970054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,15,0.050053333242734276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,15,0.04630400240421295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,15,0.04584533472855886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,31,0.04980800052483877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,31,0.05006400247414907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,31,0.0461760014295578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,31,0.04618666569391886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,63,0.05026666820049286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,63,0.05017599960168203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,63,0.04604266583919525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,63,0.0479360024134318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,127,0.05022400120894114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,127,0.050010666251182556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,127,0.04727466901143392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1,0.01584533353646596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,15,0.04981866478919983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,3,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,3,0.014736000448465347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,3,0.013781332721312841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,7,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,7,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,7,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,7,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,15,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1,0.01393066719174385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,3,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,15,0.015482666591803232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,31,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,31,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,31,0.01613866661985715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,31,0.016714667280515034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,63,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,63,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,15,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,63,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,127,0.016783999900023144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,127,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,127,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,127,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,255,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,255,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,127,0.04771733283996582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,255,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,63,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,15,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,511,0.016074666132529575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,511,0.025568000972270966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1023,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1023,0.04195733368396759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1023,0.015962666521469753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1023,0.03955200066169103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,2047,0.017429333180189133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,511,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,511,0.027232001225153606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,2047,0.06871466835339864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,4095,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,4095,0.12987732887268066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,4095,0.016373333831628162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,255,0.016229332735141117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,8191,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,2047,0.07054399947325389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,2047,0.016879999389251072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,8191,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,8191,0.2453226645787557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,16383,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,16383,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,16383,0.500058650970459
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,4095,0.12808533509572348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1,0.014650666465361914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,1,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,3,0.013839999834696451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,3,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,3,0.014394666999578476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,3,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,7,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,7,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,7,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,7,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,15,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,15,0.013616000612576803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,15,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,31,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,16383,0.5049973328908285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,31,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,31,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,63,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,63,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,63,0.015450666348139444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,15,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,63,0.015962666521469753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,127,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,127,0.017103999853134155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,31,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,127,0.015941333025693893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,8191,0.24644800027211508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,255,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,255,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,255,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,511,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,511,0.027114666998386383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,511,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,511,0.02703999976317088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1023,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,255,0.01747200017174085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,1023,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1023,0.040031999349594116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,2047,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,2047,0.07077333331108093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,2047,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,2047,0.07025066514809926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,4095,0.01758933315674464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,127,0.016506666938463848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,4095,0.016085332880417507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,4095,0.12796800335248312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1023,0.041589332123597465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,8191,0.2490453322728475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,8191,0.24485333760579428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,16383,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,4095,0.13026133179664612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,16383,0.5043520132700602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,16383,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,8191,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,8191,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,3,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,7,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,16383,0.5072480042775472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,31,0.011589333415031433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,63,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,63,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,255,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,255,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,255,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,511,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,511,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,511,0.01156266654531161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1023,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1023,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1023,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1023,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,2047,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,2047,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,4095,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,4095,0.021013334393501282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,4095,0.014015999933083853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,4095,0.019685332973798115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,2047,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,8191,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,8191,0.025077333052953083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,16383,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,16383,0.04301866888999939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,8191,0.02611200014750163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,16383,0.033674667278925575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,1,0.940554698308309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,1,0.9375786781311035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,8191,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,1,0.7508853276570638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,1,0.7483893235524496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,3,0.9388000170389811
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,3,0.7709440390268961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,16383,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,3,0.7484320004781088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,7,0.9694399833679199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,7,0.9709280331929525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,7,0.7843999862670898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,3,0.9416159788767496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,7,0.7842079798380533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,15,0.9922933578491211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,15,0.9928853511810303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,15,0.8196106751759847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,15,0.8188640276590983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,31,1.1722826957702637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,31,1.2025386492411296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,31,1.0481812953948975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,31,1.0507413546244304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,63,1.2143893241882324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,63,1.2155413627624512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,63,1.0354080200195312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,63,1.0562079747517903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,1,0.938805341720581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,1,0.9395093123118082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,1,0.7589120070139567
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,1,0.7613493601481119
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,3,0.9385440349578857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,3,0.9387413660685221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,3,0.7865066528320312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,3,0.756767988204956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,7,0.9706133206685384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,7,0.9710346857706705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,7,0.7857600053151449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,7,0.786736011505127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,15,0.990559975306193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,15,0.9904212951660156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,15,0.8205333550771078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,15,0.8238293329874674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,31,1.2029386361440022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,31,1.2046773433685303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,31,1.020026683807373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,63,1.2165066401163738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,63,1.2171306610107422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,31,1.0477226575215657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,63,1.056890646616618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,1,0.09109333157539368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,1,0.08498666683832805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,63,1.056495984395345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,1,0.0827893316745758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,3,0.09112000465393066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,1,0.09160533547401428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,3,0.08286933104197185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,3,0.08303466439247131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,7,0.09171199798583984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,7,0.09153067072232564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,7,0.08270933230717976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,7,0.08275733391443889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,15,0.09122133255004883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,15,0.09316800038019817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,15,0.08428266644477844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,15,0.08290133376916249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,31,0.09236266215642293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,31,0.09281599521636963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,31,0.08319466809431712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,31,0.08349333206812541
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,63,0.09312533338864644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,63,0.09073066711425781
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,3,0.09099733829498291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,63,0.08269333342711131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,63,0.08411733309427898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,1,1.8657387097676594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,1,1.8645920753479004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,1,1.4868052800496419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,3,1.866495927174886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,1,1.4843039512634277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,3,1.4882666269938152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,3,1.485306739807129
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,3,1.8659946123758953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,7,1.9337387084960938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,7,1.5500693321228027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,7,1.5202293395996094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,15,1.9754133224487305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,15,1.973866621653239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,15,1.6280639966328938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,15,1.630832036336263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,7,1.9318240483601887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,31,2.397717316945394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,31,2.3955839474995932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,31,2.0866880416870117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,31,2.088912010192871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,1,1.508629322052002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,1,1.5079199473063152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,1,1.86735995610555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,3,1.8709707260131836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,1,1.8682026863098145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,3,1.5084853172302246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,3,1.508885383605957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,3,1.8707733154296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,7,1.9351040522257488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,7,1.565999984741211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,7,1.9373812675476074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,7,1.5631573994954426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,15,1.994933287302653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,15,1.6348907152811687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,15,1.6363627115885417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,31,2.466970602671305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,31,2.4668426513671875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,15,1.998949368794759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,1,0.1733013391494751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,31,2.0936907132466636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,1,0.17338667313257852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,1,0.15852800011634827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,1,0.15774400035540262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,3,0.17299199104309082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,3,0.17270400126775107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,3,0.1576533317565918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,3,0.15877866744995117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,31,2.095845381418864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,7,0.17204799254735312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,7,0.1590026617050171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,7,0.15827199816703796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,15,0.17267733812332153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,15,0.1727786660194397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,15,0.15873600045839945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,15,0.15799466768900552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,31,0.17166932423909506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,31,0.17293866475423178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1,0.023077333966890972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1,0.02221333235502243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,31,0.15749333302179971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1,0.0194560003777345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,3,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,7,0.17268800735473633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,3,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,3,0.019391999890406925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,7,0.02332266668478648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,7,0.019402666638294857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,7,0.019738666713237762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,15,0.02349333216746648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,15,0.023370665808518726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,15,0.019573333362738293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,15,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,31,0.15812800327936807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,31,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,7,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,31,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,31,0.023706667125225067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,63,0.026234666506449383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,63,0.025605333348115284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,63,0.02351466566324234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,63,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,127,0.027285332481066387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,127,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,127,0.02350933353106181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,127,0.023578666150569916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,255,0.027600000301996868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,255,0.03070399910211563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,255,0.023589332898457844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,255,0.027669332921504974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,511,0.027242665489514668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,511,0.04355733096599579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,511,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,511,0.04155733436346054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1023,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1023,0.0703893353541692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1023,0.023770667612552643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1023,0.0672106643517812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,2047,0.027503999571005504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,2047,0.1218293309211731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,2047,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,2047,0.11979200442632039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,4095,0.027562665442625683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,4095,0.22684266169865927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,3,0.02218666672706604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,4095,0.22443199157714844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,31,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,8191,0.02733866622050603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,8191,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,8191,0.4378453493118286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,1,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,3,0.02165866643190384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,3,0.021877333521842957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,3,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,3,0.019013332823912304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,7,0.023152001202106476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,7,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,4095,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,7,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,15,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,15,0.020400000115235645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,15,0.02011200040578842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,31,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,31,0.02720000098148982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,31,0.02496533344189326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,31,0.02518933266401291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,63,0.027189334233601887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,63,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,7,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,63,0.023520000278949738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,127,0.025253333151340485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,127,0.025637333591779072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,127,0.023541333774725597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,8191,0.43381865819295246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,127,0.023706667125225067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,255,0.02720533311367035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,255,0.03120533376932144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,255,0.023925334215164185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,255,0.02755733331044515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,511,0.02736533433198929
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,511,0.04398400088151296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,511,0.023567999402681988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,511,0.04200000067551931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1023,0.027301333844661713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1023,0.07062399884064992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,1023,0.025370667378107708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,63,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,15,0.022997332115968067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,2047,0.12169067064921062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,2047,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,2047,0.11972799897193909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,4095,0.02624000112215678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,4095,0.22698666652043661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,4095,0.025514667232831318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,4095,0.22428266207377115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,8191,0.026650667190551758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1023,0.06826133529345195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,8191,0.023749334116776783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,8191,0.43378134568532306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,3,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,3,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,3,0.011616000284751257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,7,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,63,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,63,0.011637333780527115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,127,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,127,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,255,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,255,0.012357333054145178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,511,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,511,0.011711999773979187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1023,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1023,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,2047,0.013482666263977686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,2047,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,2047,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,2047,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,4095,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,4095,0.02515733242034912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,4095,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,4095,0.023120000958442688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,8191,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,8191,0.041589332123597465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,8191,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,8191,0.03369066615899404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1,0.03587199995915095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1,0.03130666663249334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,3,0.035775999228159584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,3,0.032629333436489105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,7,0.037205333511034645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,7,0.0332640012105306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,15,0.04342933495839437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,15,0.03952533255020777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,31,0.04341333111127218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,31,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,63,0.04372266431649526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,63,0.039488000174363456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,127,0.053344001372655235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,127,0.047695999344189964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,255,0.07654400169849396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,255,0.07251733541488647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,511,0.12568533420562744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,511,0.11964799960454305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1023,0.22406933705012003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1023,0.2140373388926188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,2047,0.42007466157277423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,2047,0.4045066833496094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,4095,0.812613328297933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,2047,0.0276053324341774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,4095,0.7837920188903809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1,0.03160533308982849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,3,0.03577066709597906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,3,0.03170666595300039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,7,0.03602133442958196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,7,0.03363733241955439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,15,0.044010668992996216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,15,0.041722665230433144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,8191,0.4405493338902791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1,0.035631999373435974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,63,0.04420266548792521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,63,0.04014399896065394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,127,0.051818668842315674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,127,0.04901333153247833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,255,0.07321066657702129
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,511,0.12643200159072876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,511,0.12051733334859212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,31,0.03975466638803482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1023,0.224565327167511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1023,0.2146880030632019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,2047,0.42563732465108234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,2047,0.40507733821868896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,255,0.07623999814192454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1,0.013610667238632837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,4095,0.8371093273162842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,4095,0.7950026988983154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,3,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,3,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,7,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,7,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,15,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,15,0.013647999614477158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,31,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,31,0.013503999759753546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,63,0.013733333597580591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,63,0.015397333850463232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,127,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,127,0.013872000078360239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,255,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,255,0.013712000101804733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,511,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,511,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1,0.015605332950750986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1023,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,2047,0.0394400010506312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,2047,0.027109332382678986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,4095,0.0603413333495458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,4095,0.04181866844495138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1,0.00938666673998038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,3,0.009077333534757296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,7,0.009285333255926767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,7,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,15,0.00926399976015091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,31,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,63,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,63,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,127,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,127,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,255,0.01952533299724261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,511,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,511,0.027232001225153606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1023,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1023,0.03138133386770884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,2047,0.040005333721637726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,2047,0.039994666973749794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,4095,0.0621973325808843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,4095,0.0584746648867925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,8191,0.10724266370137532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,8191,0.09524800380071004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,16383,0.19484267632166544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1,0.009077333534757296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,3,0.00902399979531765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,31,0.04424533247947693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,3,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,7,0.009173333023985228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,15,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,63,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,63,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,255,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,16383,0.1689173380533854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1023,0.020341333001852036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1023,0.029301332930723827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1023,0.029509333272775013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,2047,0.03978666663169861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,2047,0.03955733279387156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,4095,0.06234133243560791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,255,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,511,0.02329600105683009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,4095,0.05973333120346069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,8191,0.10764799515406291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,8191,0.09528000156084697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,16383,0.1950613260269165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,16383,0.1688800056775411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,7,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,7,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,15,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1023,0.012682666381200155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,2047,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,2047,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,4095,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,255,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,8191,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,8191,0.01764800027012825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,16383,0.021290667355060577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1,0.016800000021855038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,3,0.013354666531085968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,3,0.01681600014368693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,511,0.02548266698916753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,7,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,15,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,15,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,31,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,16383,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,31,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,63,0.01534933348496755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,63,0.020970667401949566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,127,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,127,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,255,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,255,0.021130666136741638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,511,0.021317332983016968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,511,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1023,0.0313226655125618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1023,0.03324799984693527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,2047,0.052095999320348106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,2047,0.05006400247414907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,4095,0.09107733766237895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,4095,0.08798399567604065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,8191,0.17030932505925497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,8191,0.15663466850916544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,16383,0.3264960050582886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,16383,0.29821866750717163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,3,0.015333333363135656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,7,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,7,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,15,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,15,0.01681600014368693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,31,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,31,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,63,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,63,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,127,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,127,0.020960000654061634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,255,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,255,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,511,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,511,0.023445333043734234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1023,0.031701333820819855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1023,0.03316800047953924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,2047,0.05198400219281515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,4095,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,2047,0.05012266834576925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,3,0.012714666624863943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,4095,0.08682666222254436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,8191,0.17054933309555054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,8191,0.15666133165359497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,16383,0.32757866382598877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,16383,0.29791466395060223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,7,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,15,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,31,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,63,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,4095,0.09114666779836018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1023,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,2047,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,2047,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,4095,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,4095,0.01762666677435239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,8191,0.01982933282852173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,16383,0.023578666150569916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,16383,0.021877333521842957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1,0.06461333235104878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1,0.05475200215975443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,3,0.06632000207901001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,3,0.05641066531340281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,7,0.06638399759928386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,7,0.05874133110046387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,15,0.08067200084527333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,15,0.07267733414967854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,31,0.08085333307584126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,31,0.07276799778143565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,63,0.08091199894746144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,63,0.07313066720962524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,127,0.0953493316968282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,127,0.08874133229255676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,255,0.14265599846839905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,255,0.13430399696032205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,511,0.23674132426579794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,511,0.22699199120203653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,8191,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1023,0.42869865894317627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1023,0.4107360045115153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1,0.0645546664794286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,15,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1,0.05580799778302511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,3,0.0662720004717509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,7,0.06663466493288676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,7,0.059717332323392235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,15,0.08070399860541026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,15,0.07250666618347168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,31,0.0804746647675832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,31,0.0722453345855077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,2047,0.8068319956461588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,2047,0.7754560311635336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,63,0.07238933444023132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,127,0.09547199805577596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,127,0.08879466851552327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,255,0.1425386667251587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,255,0.13414399822553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,511,0.23677867650985718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,511,0.22617600361506143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1023,0.43649065494537354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1023,0.40854934851328534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,63,0.08063466846942902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,2047,0.8304320176442465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,3,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,3,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,7,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,7,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,15,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,15,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,31,0.01855466639002164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,31,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,63,0.018794666975736618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,63,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,127,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,127,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,255,0.017504000415404636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,255,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,511,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,511,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1023,0.03633599976698557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1023,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,2047,0.06057600180308024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,2047,0.043141335248947144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,3,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,3,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,7,0.014762666076421738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,15,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,15,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,31,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,31,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,63,0.0210506667693456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,63,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,127,0.020970667401949566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,127,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,255,0.02292799949645996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,255,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,511,0.031189332405726116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,3,0.05640000104904175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,7,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1023,0.0458186666170756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1023,0.04153066625197729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,2047,0.07634666562080383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,2047,0.07031466563542683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,4095,0.1360586682955424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,4095,0.12598400314648947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,2047,0.782853364944458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,8191,0.2548533280690511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,8191,0.23757867018381754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,16383,0.4890400171279907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,16383,0.4614666700363159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,7,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,7,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,15,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,15,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,511,0.027119999130566914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,31,0.019567999988794327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,31,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,63,0.019999999552965164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,63,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,127,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,127,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,255,0.023045333723227184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,255,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,511,0.03126933425664902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,511,0.02716800073782603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1023,0.04594666759173075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1023,0.04171200096607208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,2047,0.07528000076611836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,2047,0.06877333422501881
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,4095,0.1341546674569448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,4095,0.12615467111269632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,8191,0.2531200051307678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,8191,0.23865065972010294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,16383,0.5014506578445435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,16383,0.4644426504770915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,3,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,7,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,31,0.011551999797423681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,3,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1023,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,2047,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,2047,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,4095,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,4095,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,8191,0.021226666867733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,8191,0.020053333292404812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,16383,0.03788266579310099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,16383,0.02554133286078771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1,0.12166399757067363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1,0.10122666756312053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,3,0.12496532996495564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,3,0.10526399811108907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,7,0.12785067160924277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,7,0.10945066809654236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,15,0.15452266732851663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,15,0.13802666465441385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,31,0.15425067146619162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,31,0.13795199990272522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,511,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,63,0.13807466626167297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1023,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,127,0.18360000848770142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,127,0.16868799924850464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,255,0.2773173252741496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,255,0.25917865832646686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,511,0.46293334166208905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,511,0.44380799929300946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1,0.12190933028856914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1023,0.7986133098602295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,3,0.12581866979599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,63,0.15527466932932535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,3,0.10947199662526448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,7,0.1278986632823944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,7,0.10977066556612651
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,15,0.15452266732851663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,15,0.1381653348604838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,31,0.15460800131162009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,31,0.13822399576505026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,63,0.15635200341542563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,63,0.13823999961217245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,127,0.18343466520309448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1,0.10354133447011311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,127,0.16878400246302286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,255,0.2779253323872884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,255,0.2589919964472453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,511,0.476911981900533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,511,0.44143998622894287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1023,0.8701866467793783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1023,0.7960906823476156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,3,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,3,0.021536000072956085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,7,0.02499199906984965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,7,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,15,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,31,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,31,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,63,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,63,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,127,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,127,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,255,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,255,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,511,0.03749333322048187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1023,0.05681066711743673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1023,0.03994133323431015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,1,0.2364799976348877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,1,0.19659733772277832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,3,0.24411199490229288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,3,0.20388267437616983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,7,0.247871994972229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,7,0.21427200237909952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,15,0.3020799954732259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,15,0.26899733146031696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,511,0.027669332921504974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,31,0.3023040095965068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,31,0.2681493361790975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,63,0.304474671681722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,127,0.3590826590855916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,127,0.3285973270734151
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1023,0.8384746710459391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,255,0.5442613363265991
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,255,0.5099680026372274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,3,0.24495466550191244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,63,0.2674506704012553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,7,0.2483946681022644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,7,0.2141546607017517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,1,0.23641065756479898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,15,0.30188800891240436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,15,0.2690666715304057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,31,0.3020053307215373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,3,0.20408533016840616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,63,0.303930660088857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,63,0.26919466257095337
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,127,0.3596693277359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,127,0.32844799757003784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,1,0.1977226734161377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,255,0.5653226772944132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,1,0.038133333126703896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,1,0.03160000095764796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,3,0.03782399992148081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,3,0.03178666780392329
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,7,0.03777066618204117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,7,0.03206400076548258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,15,0.037952000896135964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,15,0.03303466737270355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,31,0.03905066599448522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,31,0.03366400053103765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,63,0.03772799919048945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,63,0.03260799994071325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,255,0.5111680030822754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,127,0.03223466624816259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,255,0.03956799954175949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,255,0.03299200038115183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,1,0.4659413496653239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,3,0.4812106688817342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,127,0.038005332152048744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,3,0.4004053274790446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,31,0.2692213257153829
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,7,0.492789347966512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,7,0.4211413462956746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,1,0.38384000460306805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,15,0.5971413453420004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,15,0.5301226774851481
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,31,0.6220159928003947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,63,0.6011039813359579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,63,0.535045345624288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,127,0.7115999857584635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,127,0.650821328163147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,1,0.3866933186848958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,3,0.483791987101237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,31,0.5297386646270752
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,7,0.4936586618423462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,7,0.42101867993672687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,1,0.4621653159459432
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,15,0.5296800136566162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,31,0.5990293423334757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,3,0.401311993598938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,63,0.6220693190892538
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,15,0.5991093317667643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,63,0.5359679857889811
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,127,0.7526346842447916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,31,0.5303413470586141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,1,0.06477333108584087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,1,0.053904001911481224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,3,0.06492266555627187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,3,0.053898667295773826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,7,0.0647680014371872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,7,0.05392000079154968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,15,0.0645066648721695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,15,0.05385066568851471
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,31,0.0649599979321162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,31,0.05398400127887726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,63,0.06637866795063019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,63,0.054197331269582115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,127,0.0664106657107671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1,0.014261333892742792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,3,0.013381333400805792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,127,0.055888002117474876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,7,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,7,0.013936000565687815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,15,0.015599999576807022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,31,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,31,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,63,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,63,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,127,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,127,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,255,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,511,0.04147200038035711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,511,0.03984000037113825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,127,0.6480000019073486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1023,0.06874666611353557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,2047,0.12795733412106833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,2047,0.12593066692352295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,4095,0.24468799432118735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,255,0.025055999557177227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,4095,0.24060799678166708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,8191,0.47630401452382404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1023,0.07042133311430614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,16383,0.9759519894917806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,16383,1.0172053178151448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,3,0.013349333157142004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,3,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,7,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,7,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,8191,0.4704586664835612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,15,0.015893333901961643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,31,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,31,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,63,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,127,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,127,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,15,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,255,0.02548266698916753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,511,0.041519999504089355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,15,0.016330666840076447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,511,0.03977599988381068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1023,0.07035733262697856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1023,0.06995733578999837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,2047,0.12785599629084268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,2047,0.1272533337275187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,4095,0.24452799558639526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,63,0.016832000265518825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,8191,0.47299734751383465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,8191,0.47147198518117267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,16383,1.0722613334655762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,16383,1.0119840304056804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,4095,0.24078933397928873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,15,0.012362666428089142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,31,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,127,0.011792000383138657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,511,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1023,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1023,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,2047,0.01754666616519292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,2047,0.017157333592573803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,4095,0.021344001094500225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,4095,0.020960000654061634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,8191,0.03849600007136663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,8191,0.026346666117509205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,16383,0.05965333183606466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,16383,0.04154666761557261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,7,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,1,0.9213226636250814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,255,0.025754667818546295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,1,0.7608586947123209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,3,0.9570666948954264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,3,0.791424036026001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,7,0.9691200256347656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,7,0.8339200019836426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,15,1.1860640048980713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,15,1.047279993693034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,31,1.187712033589681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,31,1.0528799692789714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,63,1.196031967798869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,1,0.9164586861928304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,1,0.765221357345581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,3,0.9592373371124268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,3,0.7947999636332194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,7,0.9790986378987631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,15,1.1965493361155193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,15,1.0513333479563396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,7,0.8348639806111654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,31,1.2192213535308838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,31,1.0627466837565105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,1,0.11995733777681987
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,1,0.09922132889429729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,63,1.0579840342203777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,3,0.11982400218645732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,63,1.0946026643117268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,7,0.1195199986298879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,7,0.0988213320573171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,15,0.11934933066368103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,15,0.09903466701507568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,31,0.11912000179290771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,63,1.2470666567484539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,31,0.09750399986902873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,63,0.11846933762232463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,63,0.09855467081069946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,3,0.09969600041707356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,1,1.9622027079264324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,1,1.6211466789245605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,3,2.013871987660726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,3,1.6570080121358235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,7,1.7341814041137695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,15,2.431466738382975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,7,2.0532639821370444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,15,2.1454346974690757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,31,2.4359466234842935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,31,2.1574880282084146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,1,1.6729226112365723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,3,2.091957410176595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,3,1.705311934153239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,1,2.040320078531901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,7,2.1252214113871255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,7,1.7779520352681477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,15,2.4644692738850913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,15,2.1725279490152993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,31,2.4669973055521646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,1,0.18660267194112143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,3,0.22665067513783774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,3,0.1832853356997172
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,7,0.22637333472569784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,7,0.1836479902267456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,15,0.22632000843683878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,31,2.184885342915853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,15,0.18364266554514566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1,0.021274665991465252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1,0.019007999449968338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,31,0.2260106603304545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,31,0.18361600240071616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,3,0.019674666225910187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,7,0.022431999444961548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,7,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,15,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,15,0.02369600037733714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,31,0.025429333249727886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,31,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,63,0.02586666742960612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,63,0.023434666295846302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,127,0.029680001238981884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,127,0.02787200113137563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,255,0.04386133452256521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,255,0.041482667128245033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,511,0.07128533224264781
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,511,0.06675733129183452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,3,0.022111999491850536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1023,0.11772800485293071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,2047,0.22591465711593628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,2047,0.22020266453425089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,1,0.22408533096313477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,4095,0.42265601952870685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,8191,0.8491199811299642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1,0.02143466720978419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1,0.019189332922299702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,3,0.022495999932289124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,3,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,4095,0.4336106777191162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,7,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,15,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,8191,0.831658681233724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,15,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,31,0.02553066611289978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,31,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,63,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,7,0.02146133283774058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,127,0.029338667790095013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,127,0.027621333797772724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,255,0.0436106671889623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,255,0.04182399809360504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1023,0.12160533666610718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,511,0.06843199829260509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,511,0.06844800213972728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1023,0.12145599722862244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1023,0.11774399876594543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,2047,0.225983997186025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,2047,0.22016000747680664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,4095,0.4352586666742961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,4095,0.4249333143234253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1,0.011706666400035223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,8191,0.8364906311035156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,3,0.0116799995303154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,3,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,7,0.011578666667143503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,63,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,31,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,63,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,127,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,127,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,255,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,511,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1023,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1023,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,2047,0.02145066608985265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,2047,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,4095,0.03986666599909464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,4095,0.025616000096003216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,8191,0.05845333139101664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,8191,0.04161600023508072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1,0.023498666783173878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1,0.023423999547958374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,3,0.02759466568628947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,3,0.027488000690937042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,8191,0.8681973616282145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,3,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,7,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,7,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,7,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,15,0.027488000690937042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,15,0.02758399893840154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,15,0.023573334018389385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,15,0.02370133250951767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,31,0.03179199993610382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,31,0.03164800008138021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,31,0.02934933453798294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,31,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,63,0.0314026673634847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,63,0.03324266771475474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,3,0.02333866556485494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,63,0.029845332105954487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,127,0.03344533344109853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,127,0.033376000821590424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,127,0.02979733298222224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,255,0.03170666595300039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,255,0.0377813329299291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,255,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,255,0.033439998825391136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,511,0.03155199935038885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,511,0.0543146679798762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,63,0.029498666524887085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,511,0.02920000006755193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,511,0.05011733373006185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1023,0.031680000325044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1023,0.0869599978129069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1023,0.029493334392706554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1023,0.08313600222269694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,2047,0.033359999457995095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,2047,0.15249600013097128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,2047,0.15030933419863382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,4095,0.03145066648721695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,4095,0.28596266110738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,7,0.027471999327341717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,4095,0.02924799919128418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,127,0.029904000461101532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,4095,0.28197866678237915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,8191,0.03505066782236099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,8191,0.029178666571776073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,8191,0.5525759855906168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1,0.02743999908367793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1,0.027424000203609467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,3,0.027349332968393963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,3,0.02717333287000656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,3,0.023205332458019257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,3,0.02348800003528595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,7,0.02749866743882497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,2047,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,7,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,7,0.023418667415777843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,15,0.027210667729377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,15,0.027621333797772724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,15,0.024005333582560223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,15,0.02351466566324234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,31,0.031770666440327965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,31,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,7,0.027765333652496338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,31,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,63,0.032933334509531655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,63,0.029685333371162415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,63,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,127,0.0322826678554217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,127,0.03293866664171219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,127,0.02956266701221466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,127,0.0295413335164388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,31,0.029279999434947968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,255,0.03292266776164373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,8191,0.5462186733881632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,255,0.029493334392706554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,255,0.03383466601371765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,63,0.034048000971476235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,511,0.030426666140556335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,511,0.05212266743183136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1023,0.03390933324893316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1023,0.08749333024024963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1023,0.029861333469549816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1023,0.08506666620572408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,511,0.0335413341720899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,511,0.054287999868392944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,2047,0.15456533432006836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,2047,0.029658667743206024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,2047,0.15034133195877075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,4095,0.033589333295822144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,4095,0.2868746717770894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,4095,0.26773866017659503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,255,0.037685332198937736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,8191,0.0335359995563825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,2047,0.033615998923778534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,8191,0.029882666965325672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,4095,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,7,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,15,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,31,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,31,0.011648000528415045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,127,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,255,0.011637333780527115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,511,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,511,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,8191,0.5485226710637411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1023,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1023,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1023,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1023,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,2047,0.013818666338920593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,2047,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,2047,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,2047,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,4095,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,4095,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,4095,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,4095,0.02346666653951009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,8191,0.023258666197458904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,8191,0.041840001940727234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,8191,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,8191,0.03369600077470144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,8191,0.5596533219019572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,3,0.011616000284751257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,3,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,3,0.01257066677014033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,3,0.012256000190973282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,7,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,7,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,7,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,7,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,511,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,15,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,15,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,31,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,31,0.013786666095256805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,63,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1,0.01219733307758967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,63,0.01258133351802826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,63,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,127,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,15,0.012448000411192576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,127,0.014208000153303146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,127,0.012671999633312225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,127,0.012602667013804117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,31,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,255,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,255,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,255,0.01210133358836174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,511,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,511,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1023,0.012650666137536367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1023,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1023,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1023,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,2047,0.02513599892457326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,2047,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,2047,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,4095,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,255,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,4095,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,4095,0.03156800071398417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,8191,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,8191,0.06030400097370148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,8191,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,8191,0.0531626691420873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1,0.008943999807039896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1,0.008799999952316284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,3,0.009178666397929192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,3,0.008826666822036108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,7,0.009093333035707474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,15,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,15,0.008799999952316284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,31,0.00914666677514712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,63,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,31,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,63,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,63,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,127,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,127,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,127,0.013418667018413544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,255,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,31,0.008762666955590248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,255,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,511,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,511,0.013450667262077332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,4095,0.039434666434923805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1023,0.020997333029905956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1023,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1023,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,2047,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,2047,0.025914666553338368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,2047,0.022117334107557934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,2047,0.030986666679382324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,4095,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,4095,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,4095,0.02146666745344798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,4095,0.03530666728814443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,8191,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,8191,0.04766400158405304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,8191,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,8191,0.047653332352638245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,16383,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,16383,0.07690666615962982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,16383,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,16383,0.07275733351707458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,32767,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,32767,0.1349493364493052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,32767,0.021274665991465252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1,0.009045333291093508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1,0.008799999952316284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,3,0.008799999952316284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,3,0.009194666519761086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,7,0.01002133327225844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,7,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,7,0.012175999581813812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,15,0.009429333110650381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,15,0.009530666594703993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,15,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,31,0.009503999724984169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1023,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,7,0.00878399983048439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,31,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,127,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,127,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,31,0.009568000212311745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,255,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,32767,0.12160000205039978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,511,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,511,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1023,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1023,0.02125866711139679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1023,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1023,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,2047,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,2047,0.025472000241279602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,2047,0.021946666141351063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,2047,0.029264000554879505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,4095,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,4095,0.03319466610749563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,4095,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,4095,0.03552533437808355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,8191,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,8191,0.04811733464399973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,8191,0.021482666333516438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,8191,0.047637333472569786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,16383,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,16383,0.076773335536321
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,16383,0.021525333325068157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,16383,0.07262933254241943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,32767,0.019007999449968338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,32767,0.13596266508102417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,32767,0.021354667842388153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,3,0.011637333780527115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,3,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,7,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,32767,0.12168533603350322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,7,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,15,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,255,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,255,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,511,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,511,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,511,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,511,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1023,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,2047,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,2047,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,2047,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,2047,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,4095,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,4095,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,4095,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,8191,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,8191,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,8191,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,16383,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,16383,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,4095,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,16383,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,16383,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,8191,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,32767,0.020949333906173706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,32767,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,3,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,7,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,15,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,32767,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,31,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,511,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,511,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1023,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1023,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,2047,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,2047,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,2047,0.012597333639860153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,4095,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,4095,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,4095,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,4095,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,8191,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,8191,0.015354666858911514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,8191,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,8191,0.015466666469971338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,16383,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,16383,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,16383,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,32767,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,32767,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,32767,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,32767,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1,0.00915733352303505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1,0.009183999771873156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,3,0.00879466657837232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,3,0.008842666943868002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,16383,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,3,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,7,0.009039999917149544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,7,0.00926399976015091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,15,0.009365333244204521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,15,0.009258666386206945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,31,0.009818666925032934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,31,0.009957333405812582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,32767,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,127,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,255,0.012554666648308435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,255,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,511,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,511,0.020954666038354237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,511,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,511,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1023,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1023,0.023013333479563396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1023,0.02314666658639908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1023,0.027285332481066387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,2047,0.019893333315849304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,2047,0.02914133419593175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,2047,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,2047,0.03143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,4095,0.01964266722400983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,4095,0.04159466673930486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,4095,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,4095,0.04141333450873693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,8191,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,8191,0.062261333068211876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,8191,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,8191,0.05994666616121928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,16383,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,16383,0.1072266697883606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,16383,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,127,0.013530666629473368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,16383,0.09705600142478943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,32767,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,32767,0.19598400592803955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1,0.009450666606426239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,32767,0.1708959937095642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1,0.009296000003814697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,3,0.00901333304742972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,3,0.009312000125646591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,3,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,7,0.008869333192706108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,15,0.008912000184257826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,31,0.009589333087205887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,31,0.009141333401203156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,31,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,15,0.00890666681031386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,7,0.008943999807039896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,255,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,255,0.010533332824707031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,255,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,511,0.019733333339293797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,511,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,511,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,127,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1023,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1023,0.02309866746266683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,32767,0.02317333221435547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1023,0.023178666830062866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1023,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,63,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,2047,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,2047,0.03123733401298523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,4095,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,511,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,4095,0.023589332898457844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,2047,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,2047,0.029743999242782593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,4095,0.040336000422636666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,8191,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,8191,0.06206400195757548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,8191,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,8191,0.060592000683148704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,16383,0.019189332922299702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,16383,0.10733866691589355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,16383,0.02312533309062322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,16383,0.09705066680908203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,32767,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,32767,0.19565866390864053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,32767,0.023232000569502514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,32767,0.17082132895787558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,4095,0.04139200101296107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,3,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,3,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,31,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,63,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,63,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,255,0.011642667154471079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,511,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,511,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1023,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1023,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,2047,0.011551999797423681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,2047,0.01339200014869372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,7,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,2047,0.01181866725285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,4095,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,4095,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,4095,0.011802667131026586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,8191,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,8191,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,8191,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,8191,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,2047,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,16383,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,16383,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,16383,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,4095,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,16383,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,15,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,32767,0.021055998901526134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,3,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,32767,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,7,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,15,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,7,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,32767,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,32767,0.020682666450738907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,63,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,127,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,255,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,511,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,511,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,511,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,2047,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,2047,0.013418667018413544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,2047,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,4095,0.011829332758982977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,4095,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,255,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,8191,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,8191,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,8191,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,8191,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,16383,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,16383,0.020538666596015293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,16383,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,16383,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,32767,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,32767,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,4095,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,32767,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,32767,0.02314666658639908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1,0.045653333266576133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1,0.03825066735347112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1,0.03772799919048945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,3,0.0462773342927297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,3,0.04561600089073181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,3,0.038805333276589714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,3,0.037861332297325134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,7,0.0473280002673467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,7,0.04743466774622599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,7,0.03972800076007843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,7,0.03956799954175949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,15,0.04799466828505198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,15,0.04808000226815542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,15,0.03979199876387914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,15,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,31,0.05611200133959452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,31,0.056015998125076294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,31,0.050053333242734276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,31,0.0498933345079422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,63,0.05651199817657471
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,63,0.057855998476346336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,63,0.050144001841545105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,63,0.050106664498647056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,127,0.05791999896367391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,127,0.058149332801500954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,127,0.05009600023428599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,127,0.0498986691236496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,255,0.05795733133951823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,255,0.06674133241176605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,255,0.050144001841545105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,255,0.060271998246510826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,511,0.057962665955225624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,511,0.09905067086219788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,511,0.05007466673851013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,511,0.09303466478983562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1023,0.05813866853713989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1023,0.1607253352801005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1023,0.050154666105906166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1023,0.15449066956837973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,2047,0.058143998185793556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,2047,0.2894773284594218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1,0.045978665351867676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,4095,0.05853866537412008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,4095,0.5419893264770508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,4095,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,2047,0.04985600213209788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1,0.04558399816354116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,4095,0.5374240080515543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1,0.0458186666170756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1,0.039066667358080544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1,0.03806933263937632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,3,0.04587199787298838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,3,0.046015997727712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,3,0.04168533285458883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,3,0.03826133410135905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,7,0.04790933430194855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,7,0.04804266492525736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,7,0.03968533376852671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,7,0.03969600051641464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,15,0.04931733508904775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,15,0.04776533444722494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,15,0.041989331444104515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,15,0.041749333341916404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,31,0.058117335041364036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,31,0.05611733098824819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,31,0.05206400156021118
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,31,0.05202133456865946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,63,0.0582826683918635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,63,0.05843733251094818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,63,0.051962668697039284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,4095,0.051914667089780174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,63,0.052101333936055504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,127,0.05831466615200043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,127,0.05194133520126343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,2047,0.2797813415527344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,127,0.05186666548252106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,255,0.05799466868241628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,255,0.06828799843788147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,255,0.05036800106366476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,511,0.0582239975531896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,511,0.09915199875831604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,511,0.051829333106676735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,511,0.09316800038019817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1023,0.058005332946777344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1023,0.16089066863059998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1023,0.1544533371925354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,2047,0.05825066566467285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,2047,0.28920533259709674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,127,0.0582239975531896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,2047,0.050010666251182556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,2047,0.2836479942003886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,4095,0.05793066819508871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1023,0.052255998055140175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,4095,0.5510773261388143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1,0.01267733300725619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,3,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,3,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,3,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,7,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,7,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,7,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,15,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,15,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,15,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,255,0.06039999922116598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,31,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,4095,0.532421350479126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,31,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,31,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,31,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,63,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,63,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,63,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,63,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,127,0.012703999876976013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,127,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,127,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,255,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,255,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,511,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,511,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,511,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,511,0.015439999600251516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1023,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1023,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1023,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1023,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,2047,0.016693333784739178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,2047,0.0161920003592968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,2047,0.025087999800841015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,4095,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,4095,0.04237333436806997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,4095,0.015413332730531693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,4095,0.05043200155099233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1,0.013594667116800943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1,0.013898666948080063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1,0.014111999422311783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1,0.01422400027513504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,3,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,3,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,3,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,3,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,15,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,7,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,7,0.014736000448465347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,7,0.013823999712864557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,7,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,15,0.01461333284775416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,15,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,15,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,31,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,31,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,31,0.014682666709025701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,31,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,63,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,63,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,63,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,63,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,127,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,127,0.014773332824309668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,127,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,127,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,255,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,255,0.013951999445756277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,255,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,255,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,511,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,511,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,511,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,511,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1023,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1023,0.021727999051411945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1023,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,4095,0.035461333890755974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1023,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,2047,0.017946666727463405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,2047,0.01722666621208191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,2047,0.03403199960788091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,4095,0.0621066689491272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,4095,0.054085334142049156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,3,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,2047,0.02595199892918269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,3,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,3,0.00922133338948091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,3,0.009445333232482275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,7,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,7,0.013679999858140945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,7,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,15,0.013482666263977686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,15,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,15,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,15,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,31,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,31,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,31,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,63,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,63,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,63,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,63,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,127,0.01543466622630755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,127,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,127,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,255,0.01573333392540614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,255,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,511,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,511,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1023,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1023,0.02310933421055476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1023,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1023,0.019226666539907455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,2047,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,2047,0.03326933334271113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,2047,0.02916266769170761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,4095,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,4095,0.05216533442338308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,2047,0.04035733391841253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,8191,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,8191,0.09302933017412822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,8191,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,8191,0.08721066514650981
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,511,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,16383,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,16383,0.17185600598653158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,16383,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,255,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,16383,0.16484799981117249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,32767,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,32767,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1,0.013552000125249227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,32767,0.3335893154144287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,3,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,3,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,3,0.01051733394463857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,3,0.010101333260536194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,7,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,7,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,32767,0.3306933244069417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,7,0.009088000282645226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1,0.013605333864688873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,15,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,4095,0.048165331284205117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,31,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,31,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,31,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,15,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,63,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,63,0.01532799998919169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,63,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,63,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,127,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,127,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,127,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,127,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,255,0.015482666591803232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,255,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,255,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,255,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,511,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,511,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,511,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1023,0.023082666099071503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1023,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1023,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,2047,0.016106666376193363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,2047,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,2047,0.029509333272775013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,4095,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,4095,0.052149335543314614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1023,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,4095,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,4095,0.0480320006608963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,8191,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,8191,0.09311466415723164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,8191,0.012725333372751871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,8191,0.08693333466847737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,16383,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,16383,0.17189866304397583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,16383,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,16383,0.1648906668027242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,32767,0.015594666202863058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,15,0.01357866699496905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,32767,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,2047,0.03359466542800268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,31,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,15,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,127,0.012602667013804117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,255,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,511,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,511,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,511,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1023,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1023,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1023,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1023,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,2047,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,2047,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,2047,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,4095,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,4095,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,4095,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,4095,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,8191,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,8191,0.018719999740521114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,8191,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,8191,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,16383,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,16383,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,16383,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,16383,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,32767,0.3332800070444743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,32767,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,32767,0.027664000789324444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,32767,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,32767,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,32767,0.3303520083427429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,7,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,15,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,31,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,31,0.011776000261306763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,31,0.011567999919255575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,31,0.011744000017642975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,63,0.011594666788975397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,63,0.011685332904259363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,63,0.01166933278242747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,511,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,511,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1023,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1023,0.011829332758982977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,2047,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,2047,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,2047,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,2047,0.013546666751305262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,4095,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,4095,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,4095,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,4095,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,8191,0.017664000391960144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,8191,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,8191,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,16383,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,16383,0.023258666197458904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,16383,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,16383,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,32767,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,32767,0.03789333254098892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,32767,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1,0.08507733543713887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1,0.08508800466855367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1,0.07050666709740956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1,0.07028266787528992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,3,0.08471999565760295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,3,0.08532800277074178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,3,0.07061333457628886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,32767,0.03126399964094162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,7,0.08705600102742513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,7,0.08741866548856099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,7,0.07259200016657512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,7,0.07276266813278198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,511,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,15,0.08897599577903748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,3,0.07037866612275441
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,15,0.0744053324063619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,15,0.07460266848405202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,31,0.10732266306877136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,31,0.10725333293279012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,31,0.09411733349164327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,31,0.09499733646710713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,63,0.10736533006032307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,63,0.10757333040237427
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,63,0.09493866562843323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,8191,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,127,0.10912000139554341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,127,0.10946133732795715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,127,0.09503466884295146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,127,0.09522133072217305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,255,0.10921066999435425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,255,0.12819199760754904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,255,0.0951039989789327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,255,0.115365336338679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,511,0.10916266838709514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,511,0.19144533077875772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,63,0.09502399961153667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,511,0.17732266585032144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1023,0.10959466298421223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1023,0.3124053279558818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1023,0.09526399771372478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,511,0.0950879951318105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,15,0.08914132912953694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1023,0.30001066128412884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,2047,0.10948800047238667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,2047,0.09539199868837993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,2047,0.5680586496988932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1,0.08528000116348267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1,0.08454400300979614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1,0.07043733199437459
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1,0.07053333520889282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,3,0.084906667470932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,3,0.09166399637858073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,3,0.07046400010585785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,3,0.07029333213965099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,7,0.08847999572753906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,7,0.08760533730189006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,7,0.0724373310804367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,7,0.07362666726112366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,15,0.0892639954884847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,15,0.08925333619117737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,15,0.07544533411661784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,31,0.1074133316675822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,31,0.10757866501808167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,31,0.0949173370997111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,31,0.09497599800427754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,63,0.10939199725786845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,63,0.10951466361681621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,63,0.09496532877286275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,63,0.09540800253550212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,127,0.10916266838709514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,127,0.10949333508809407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,127,0.09513066212336223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,127,0.09687999884287517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,15,0.07437866429487865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,255,0.1281066636244456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,255,0.09513599673906963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,255,0.11541333794593811
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,511,0.10935999949773152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,511,0.18992000818252563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,511,0.09485333164532979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,511,0.17800533771514893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1023,0.10957333445549011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1023,0.3136799931526184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,2047,0.5523306528727213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1023,0.30023467540740967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,2047,0.10938133796056111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,2047,0.5910880168279012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,2047,0.5500213305155436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1023,0.09513066212336223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,255,0.1095306674639384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,3,0.01551466683546702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,3,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,3,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,3,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,7,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,7,0.015594666202863058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,7,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,7,0.015429332852363586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,15,0.015530666957298914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,15,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,15,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,15,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,31,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,31,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,31,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,63,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,63,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,2047,0.09490666786829631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,63,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,127,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,127,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,127,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,127,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,255,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,255,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,255,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,255,0.015392000476519266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,511,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,511,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,511,0.01597333326935768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,511,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1023,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1023,0.025146665672461193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1023,0.015989333391189575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1023,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,2047,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,2047,0.04387199878692627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,31,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,2047,0.03551466763019562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,63,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1,0.018768000106016796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,3,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,3,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,2047,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,3,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,3,0.018330667167901993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,7,0.01871466636657715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,7,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,7,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,15,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,15,0.019013332823912304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,15,0.019007999449968338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,15,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,31,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,31,0.01886933296918869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,31,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,31,0.01794133335351944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,63,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,63,0.018677332748969395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,63,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,63,0.01756799966096878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,127,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,127,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,127,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,127,0.017711999515692394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,255,0.019167999426523846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,255,0.018122666825850803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,255,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,511,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,511,0.02327466756105423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,511,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,511,0.023034666975339253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1023,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1023,0.03616533428430557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,7,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1023,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1023,0.02934933453798294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,2047,0.02111999938885371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,2047,0.05813866853713989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,2047,0.021503999829292297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,255,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,1,0.16263999541600546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,1,0.16472533345222473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,1,0.13341866930325827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,1,0.13205867012341818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,3,0.1646613379319509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,3,0.1646293302377065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,3,0.13197867075602213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,3,0.13225066661834717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,7,0.16889599959055582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,7,0.16883732875188193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,7,0.13808000087738037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,7,0.13806933164596558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,15,0.170799990495046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,15,0.17294400930404663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,15,0.14408533771832785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,15,0.14242133498191833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,31,0.20801067352294922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,31,0.20715200901031494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,31,0.18108266592025757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,31,0.18121600151062012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,63,0.20980799198150635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,63,0.20963199933369955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,63,0.18332799275716147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,63,0.1828213334083557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,127,0.209824005762736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,127,0.21201600631078085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,127,0.18338666359583536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,127,0.18324265877405801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,255,0.21172267198562622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,255,0.24707732597986856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,255,0.1833546757698059
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,255,0.22233066956202188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,511,0.2116746703783671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,511,0.37166933218638104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,511,0.18334933121999106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,511,0.34513600667317706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,1,0.16077333688735962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,1,0.16481600205103555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,1,0.13386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,1,0.13801067074139914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,3,0.16289066274960837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,3,0.16358400384585062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,3,0.1325813333193461
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,3,0.13241599996884665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,7,0.16885334253311157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,7,0.1688800056775411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,7,0.13833066821098328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,7,0.13782933354377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,15,0.17105066776275635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,15,0.1727893352508545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,15,0.14250133434931436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,15,0.14257066448529562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,31,0.20798399051030478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,31,0.2059626579284668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,31,0.18118399381637573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,31,0.18116267522176108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,63,0.20987733205159506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,63,0.20798399051030478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,63,0.183514674504598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,2047,0.05032533407211304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,127,0.21156267325083414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,127,0.20997333526611328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,127,0.18406933546066284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,127,0.1831093430519104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,255,0.21177067359288534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,255,0.24687467018763223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,255,0.1854026714960734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,255,0.22057066361109415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,511,0.21224000056584677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,63,0.18313066164652506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,511,0.3699680169423421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,511,0.3452053467432658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,1,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,1,0.021365332106749218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,1,0.0194560003777345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,1,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,3,0.01939733326435089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,3,0.01912533367673556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,3,0.019776000330845516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,3,0.01913600042462349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,7,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,7,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,7,0.019402666638294857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,15,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,15,0.02094399929046631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,15,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,15,0.019445333629846573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,31,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,31,0.019445333629846573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,31,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,31,0.01916266605257988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,63,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,63,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,63,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,63,0.01952533299724261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,127,0.021055998901526134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,127,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,127,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,127,0.01947733387351036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,255,0.02141333371400833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,255,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,255,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,255,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,511,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,511,0.027263998985290527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,7,0.021674667795499165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,511,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,511,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,1,0.02804800122976303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,1,0.02569599946339925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,1,0.02659733345111211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,3,0.02717866748571396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,3,0.02754133443037669
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,3,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,3,0.02643733223279317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,7,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,511,0.18523732821146646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,7,0.025642665723959606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,15,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,15,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,15,0.025562666356563568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,15,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,31,0.02743999908367793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,7,0.02565866708755493
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,31,0.027450665831565857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,31,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,31,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,63,0.027232001225153606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,63,0.027386667827765148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,63,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,63,0.02685333291689555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,127,0.027210667729377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,127,0.027434666951497395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,127,0.025583999852339428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,127,0.02553066611289978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,1,0.027242665489514668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,255,0.02741866558790207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,255,0.025562666356563568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,255,0.025568000972270966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,511,0.027290667096773785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,7,0.027269333600997925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,511,0.03974399964014689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,511,0.025621332228183746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,511,0.033514666060606636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,1,0.32051199674606323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,1,0.3195786674817403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,255,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,1,0.25675733884175617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,3,0.3203093409538269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,3,0.32069865862528485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,3,0.2569599946339925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,3,0.2568800052007039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,7,0.33129066228866577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,7,0.3292959928512573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,7,0.2672160069147746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,1,0.2560266653696696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,15,0.3367573420206706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,15,0.3361599842707316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,15,0.277621328830719
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,15,0.27957866589228314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,31,0.40644800662994385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,31,0.4245813290278117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,31,0.3534506559371948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,7,0.2691626747449239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,31,0.3549813429514567
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,63,0.4105813503265381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,63,0.4126773277918498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,63,0.3577226797739665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,63,0.35735468069712323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,127,0.4150133530298869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,127,0.4126720031102498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,127,0.39031465848286945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,127,0.3594026565551758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,255,0.414741317431132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,255,0.4864266713460286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,255,0.36110933621724445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,255,0.4344213406244914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,1,0.32079466183980304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,1,0.2589866717656453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,1,0.25701866547266644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,3,0.31938666105270386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,3,0.3209173281987508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,3,0.25737067063649494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,3,0.2570026715596517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,7,0.3309226632118225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,7,0.3288106719652812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,7,0.2674186627070109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,7,0.2676159938176473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,1,0.31887465715408325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,15,0.3348960081736247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,15,0.33503464857737225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,15,0.2797653277715047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,15,0.2792746623357137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,31,0.406549334526062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,31,0.40725866953531903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,31,0.3553226788838704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,31,0.3548640012741089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,63,0.410533348719279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,63,0.4105973243713379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,63,0.3579466740290324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,63,0.35742934544881183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,127,0.4129386742909749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,127,0.41316266854604083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,127,0.3604213396708171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,127,0.35930665334065753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,255,0.41340800126393634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,255,0.48670931657155353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,255,0.4352000157038371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,1,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,1,0.031221332649389904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,1,0.029461334149042766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,1,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,3,0.031658666829268135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,3,0.03126933425664902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,3,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,3,0.029802667597929638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,7,0.031152000029881794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,7,0.03142400085926056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,7,0.030005333324273426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,7,0.029301332930723827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,15,0.03139200061559677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,15,0.03155199935038885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,15,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,15,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,31,0.03162133445342382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,31,0.03161066770553589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,31,0.029631999631722767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,31,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,63,0.031354665756225586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,63,0.031712000568707786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,63,0.029626667499542236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,63,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,255,0.3595466613769531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,127,0.03145066648721695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,127,0.029525332152843475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,127,0.02996266633272171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,255,0.03363200028737386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,255,0.03142933299144109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,255,0.031712000568707786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,255,0.029530666768550873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,1,0.04364799956480662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,1,0.04154666761557261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,1,0.04159466673930486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,3,0.04387733340263367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,3,0.04399466514587402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,3,0.04193066557248434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,3,0.04194133480389913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,7,0.044031997521718345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,1,0.0439573327700297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,7,0.04372799893220266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,7,0.04176533222198486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,7,0.04162666698296865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,15,0.04381333291530609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,127,0.03133333226044973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,15,0.04152533411979675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,15,0.0415040006240209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,31,0.04410133262475332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,31,0.0436160018046697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,31,0.04177600145339966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,31,0.04160533348719279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,15,0.0441599984963735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,63,0.044138665000597634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,63,0.04156800111134847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,127,0.04386133452256521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,63,0.044079999128977455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,127,0.04190933207670847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,63,0.04174399872620901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,255,0.04494933287302653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,255,0.04668800036112467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,127,0.04403733213742574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,255,0.0414986660083135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,3,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,127,0.04206933577855428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,3,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,3,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,7,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,7,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,7,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,255,0.04199466605981191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,15,0.013440000514189402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,15,0.01589866727590561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,3,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,15,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,31,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,31,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,31,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,31,0.01648533344268799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,63,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,63,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,63,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,63,0.019493332753578823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,127,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,127,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,127,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,127,0.019487999379634857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,255,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,255,0.015541333705186844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,255,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,255,0.01913600042462349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,511,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,511,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,511,0.01953599974513054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,511,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1023,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1023,0.031471999982992806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1023,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1023,0.03352533280849457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,2047,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,2047,0.05226666728655497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,2047,0.01985599969824155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,7,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,15,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,4095,0.09114133318265279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,4095,0.08896000186602275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,8191,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,8191,0.17318934202194214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,8191,0.019621333728233974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,8191,0.1628266672293345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,4095,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,16383,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,16383,0.3389493227005005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,16383,0.31270400683085126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,32767,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,2047,0.052000001072883606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,4095,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,32767,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,16383,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,3,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,3,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,3,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,3,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,7,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,7,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,7,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,32767,0.9070826371510824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,15,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,32767,0.6562346617380778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,15,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,15,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,31,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,31,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,31,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,31,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,63,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,7,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,63,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,63,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,63,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,127,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,127,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,127,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,127,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,255,0.01681600014368693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,255,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,255,0.019226666539907455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,511,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,511,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,511,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,511,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1023,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,15,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1023,0.031504000226656594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1023,0.019386666516462963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1023,0.033514666060606636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,2047,0.05189866820971171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,2047,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,2047,0.051829333106676735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,4095,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,4095,0.09115200241406758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,4095,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,4095,0.08889066179593404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,8191,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,8191,0.17299199104309082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,8191,0.16299200057983398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,255,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,16383,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,2047,0.015423999478419622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,16383,0.3391466538111369
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,16383,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,16383,0.3128640055656433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,32767,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,32767,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,32767,0.9285226662953695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,3,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,7,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,8191,0.019509332875410717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,63,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,32767,0.654917319615682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,15,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,511,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,511,0.012639999389648438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1023,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1023,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1023,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,2047,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,2047,0.015322666615247726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,2047,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,4095,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,4095,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,8191,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,8191,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,8191,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,16383,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,16383,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,16383,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,16383,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,32767,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,32767,0.04378133515516917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,32767,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,32767,0.03531199942032496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,3,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,8191,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,15,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,15,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,63,0.012479999413092932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,511,0.012234666695197424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1023,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1023,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,2047,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,2047,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,4095,0.018960000326236088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,4095,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,8191,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,8191,0.023258666197458904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,8191,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,8191,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,16383,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,16383,0.037690666814645134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,16383,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,16383,0.02998399982849757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,32767,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,32767,0.05826666454474131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,511,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,32767,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,32767,0.051818668842315674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,1,0.6319306691487631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,1,0.6313546498616537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,1,0.5047573248545328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,3,0.6316266854604086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,3,0.6318720181783041
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,3,0.5050026575724283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,3,0.5042133331298828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,1,0.5034666856129965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,7,0.6521120071411133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,7,0.5273120005925497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,7,0.527237335840861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,15,0.6570613384246826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,15,0.665173331896464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,15,0.5502560138702393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,15,0.5738773345947266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,7,0.6539733409881592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,31,0.686901330947876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,31,0.7033600012461344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,31,0.806874672571818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,63,0.809829314549764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,63,0.8298400243123373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,63,0.7095359961191813
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,63,0.7079306443532308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,31,0.8061546484629313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,127,0.820197343826294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,127,0.7118666966756185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,127,0.7114506562550863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,1,0.6315040191014608
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,1,0.6518453359603882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,127,0.8228906790415446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,1,0.5071359872817993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,3,0.6309173504511515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,3,0.631498654683431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,3,0.5080586671829224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,1,0.5070506731669108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,7,0.6525173187255859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,7,0.5274879932403564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,3,0.5082986752192179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,7,0.5274720191955566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,15,0.6632533470789591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,15,0.6634933153788248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,15,0.5503679911295573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,15,0.5501866738001505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,31,0.8058186372121176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,31,0.8060746987660726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,31,0.7012693087259928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,31,0.7031733194986979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,63,0.8147520224253336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,63,0.8139572938283285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,7,0.6521066824595133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,63,0.7093119621276855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,63,0.7165119647979736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,127,0.8157013257344564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,127,0.8204533259073893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,127,0.7117599646250407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,1,0.0525439977645874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,127,0.711189349492391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,1,0.048010667165120445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,1,0.04837866624196371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,3,0.05212266743183136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,3,0.05202666421731313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,3,0.048245335618654885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,3,0.049957334995269775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,7,0.05183466772238413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,7,0.05192000170548757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,7,0.04939733445644379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,1,0.05301866432030996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,7,0.050106664498647056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,15,0.05218133330345154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,15,0.05398933092753092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,15,0.04939733445644379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,15,0.04987733562787374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,31,0.052149335543314614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,31,0.05320000151793162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,31,0.049039999643961586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,31,0.04984533290068308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,63,0.05212266743183136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,63,0.0528053343296051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,63,0.04986133178075155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,63,0.05011733373006185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,127,0.05250666538874308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,127,0.05397333204746246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,127,0.048469334840774536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,127,0.049738665421803795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,1,0.07885333398977916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,1,0.0787360022465388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,1,0.07253866891066234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,1,0.07320000231266022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,3,0.07895466685295105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,3,0.0787306676308314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,3,0.07226666808128357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,3,0.07255466779073079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,7,0.07860266665617625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,7,0.07892799874146779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,7,0.07266666491826375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,7,0.07264000177383423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,15,0.07864533364772797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,15,0.07864533364772797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,15,0.07285866638024648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,15,0.0726986676454544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,31,0.07877866427103679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,31,0.07897600034872691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,31,0.07271466652552287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,31,0.07269333302974701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,63,0.07866666714350383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,63,0.08002666632334392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,63,0.07259200016657512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,63,0.072543998559316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,127,0.07853333155314128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,127,0.08009600142637889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,127,0.07457066575686137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,127,0.07234666744867961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,1,1.2546933492024739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,1,1.2531147003173828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,3,1.2550986607869465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,3,1.225610653559367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,1,1.0006240208943684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,1,0.9985386530558268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,3,0.9996373653411865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,3,1.0022666454315186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,7,1.2975573539733887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,7,1.2950026988983154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,7,1.0457759698232014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,7,1.0422826608022053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,15,1.3219199975331624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,15,1.3222239812215169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,15,1.0911040306091309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,15,1.0667413075764973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,31,1.6055893898010254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,31,1.6026560465494792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,31,1.3939785957336426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,31,1.37444273630778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,63,1.6229972839355469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,63,1.6196640332539876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,63,1.4072319666544597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,63,1.4083093007405598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,1,1.2552586396535237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,1,1.252346674601237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,1,1.006442705790202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,1,0.9818080266316732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,3,1.2554879983266194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,3,1.2526933352152507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,3,1.0069013436635335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,3,1.005674680074056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,7,1.2969813346862793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,7,1.2951786518096924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,7,1.0477279822031658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,7,1.0478346347808838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,15,1.3178826967875164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,15,1.0932532946268718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,31,1.605514685312907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,31,1.6048320134480794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,15,1.319642702738444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,31,1.3971786499023438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,15,1.093786636988322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,31,1.3956054051717122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,63,1.6329174041748047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,63,1.6348533630371094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,63,1.410410722096761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,1,0.09649067123730977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,1,0.09723200400670369
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,1,0.08861333131790161
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,1,0.08858666817347209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,3,0.09708799918492635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,63,1.4080212910970051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,3,0.09700266520182292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,3,0.08833066622416179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,3,0.08900266885757446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,7,0.09669333696365356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,7,0.09702400366465251
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,7,0.08819733063379924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,7,0.08895466725031535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,15,0.0962666670481364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,15,0.09702933828035991
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,15,0.08747200171152751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,31,0.09642666578292847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,31,0.09692800045013428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,31,0.08678932984670003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,31,0.08733333150545756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,63,0.0969599982102712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,63,0.09528000156084697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,63,0.08761599659919739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,63,0.08870399991671245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,1,0.1467466652393341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,1,0.14704533418019614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,15,0.08900266885757446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,1,0.13609600067138672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,3,0.14696533481280008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,3,0.14789332946141562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,3,0.1359999974568685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,1,0.1362826625506083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,3,0.13805333773295084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,7,0.14823466539382935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,7,0.14706133802731833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,7,0.13593600193659464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,7,0.13768000404040018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,15,0.14823466539382935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,15,0.14711999893188477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,15,0.1360373298327128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,15,0.13665599624315897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,31,0.14823466539382935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,31,0.14831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,31,0.13611732920010886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,31,0.13700266679128012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,63,0.14831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,63,0.14839466412862143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,63,0.13642133275667825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,63,0.13730133573214212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,3,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,3,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,3,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,7,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,7,0.018895999838908512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,7,0.015493333339691162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,7,0.015546667079130808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,15,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,15,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,31,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,31,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,31,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,31,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,63,0.0195573332409064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,63,0.019674666225910187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,63,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,63,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,127,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,127,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,127,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,127,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,255,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,255,0.023013333479563396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,255,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,255,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,511,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,511,0.031040000418821972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,511,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,15,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,3,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1023,0.04794666667779287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1023,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,15,0.017717332889636356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1023,0.04588800172011057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,2047,0.020101333657900494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,511,0.029317334294319153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,2047,0.08268799881140391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,2047,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,2047,0.08070399860541026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,4095,0.02012266715367635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,4095,0.15067199865976968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,4095,0.1497760017712911
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,8191,0.02089600016673406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,8191,0.28683199485143024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1023,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,8191,0.284277339776357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,4095,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,16383,0.019695999721686046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,16383,0.017621333400408428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,16383,0.591759999593099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1,0.017903999735911686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,3,0.017664000391960144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,3,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,3,0.015322666615247726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,7,0.017637333522240322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,7,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,7,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,7,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,15,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,15,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,16383,0.5994879802068075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,15,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,8191,0.017818666994571686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,3,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,31,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,31,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,31,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,63,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,63,0.019920000185569126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,63,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,63,0.017717332889636356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,127,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,127,0.021189334491888683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,127,0.01764800027012825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,127,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,255,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,255,0.02327466756105423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,255,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,255,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,511,0.020261333634455998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,511,0.0317493329445521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,511,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,511,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1023,0.0194560003777345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1023,0.047914668917655945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1023,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1023,0.04588800172011057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,2047,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,2047,0.08122133215268452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,2047,0.08086400230725606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,4095,0.020234666764736176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,15,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,4095,0.15060266852378845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,4095,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,31,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,8191,0.02049066623051961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,8191,0.28777066866556805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,8191,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,8191,0.2858133316040039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,16383,0.6072959899902344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,4095,0.14894933501879373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,16383,0.599290649096171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,2047,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,3,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,3,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,7,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,31,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,127,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,127,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,16383,0.018218666315078735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,255,0.012144000579913458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,16383,0.020181333025296528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,511,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,511,0.011834666132926941
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,511,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1023,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1023,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1023,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,2047,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,2047,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,2047,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,2047,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,4095,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,4095,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,8191,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,8191,0.025562666356563568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,8191,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,8191,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,16383,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,16383,0.041984001795450844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,4095,0.015941333025693893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,16383,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1,0.011626667032639185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,3,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,3,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,3,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,3,0.011616000284751257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,7,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,16383,0.03459733227888743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,15,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,15,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,4095,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,255,0.012026666353146235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,511,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,511,0.01163200040658315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1023,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1023,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,2047,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,2047,0.018453333526849747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,2047,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,2047,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,4095,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,4095,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,4095,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,4095,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,8191,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,8191,0.037605332831541695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,8191,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,8191,0.029552000264326733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,16383,0.05830933153629303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,16383,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,16383,0.05231466889381409
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,3,0.027130665878454845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,3,0.023317334552605946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,7,0.027141332626342773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,7,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,15,0.031386665999889374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,15,0.02943466603755951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,31,0.03140799949566523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,31,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,63,0.03140799949566523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,63,0.02922666569550832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,127,0.03705599904060364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,127,0.035274667044480644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,255,0.05400000015894572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,255,0.05003733436266581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,511,0.08695466319719951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,511,0.08284799754619598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1023,0.1524853308995565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1023,0.14814399679501852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,2047,0.28388800223668414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,2047,0.275493323802948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,4095,0.5483413139979044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,4095,0.530784010887146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,8191,1.0873013337453206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1,0.025775998830795288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1,0.02350933353106181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,3,0.027210667729377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,3,0.0242399995525678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,7,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,7,0.024442667762438457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,15,0.029919999341169994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,31,0.03189333279927572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,31,0.03027733415365219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,16383,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,63,0.03175999969244003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,63,0.02939733366171519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,127,0.03409600009520849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,255,0.05410666763782501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,255,0.05000533163547516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,15,0.03345066557327906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,511,0.08719467123349507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,511,0.08287466565767924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1023,0.15270400047302246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1023,0.1469439963499705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,2047,0.2845013340314229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,2047,0.27556800842285156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,127,0.03809066613515218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,4095,0.5522773265838623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,4095,0.5317173401514689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,8191,1.112773338953654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,8191,1.0498666763305664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,3,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,7,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,7,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,15,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,15,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,31,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,63,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,127,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,3,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,255,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,255,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,511,0.013424000392357508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,511,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1023,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1023,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,2047,0.021583999196688335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,2047,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,4095,0.03957333415746689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,4095,0.025621332228183746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,8191,0.060032000144322716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,8191,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,3,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,3,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,7,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,7,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,15,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,15,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,31,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,31,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,63,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,63,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,127,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,127,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,255,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,255,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,511,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,511,0.013445333888133367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1023,0.019226666539907455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1023,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,2047,0.039247999588648476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,2047,0.023999998966852825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,4095,0.06017066538333893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,4095,0.03926933308442434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,31,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,8191,0.10129066308339436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,8191,0.0599839985370636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,3,0.009418666362762451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,7,0.00884799969693025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,7,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,15,0.008826666822036108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,31,0.009061333412925402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,63,0.00884799969693025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,63,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,127,0.009141333401203156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,8191,1.0492693583170574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,255,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,255,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,511,0.021551998953024547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,511,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1023,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1023,0.027647999425729115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,2047,0.03379733363787333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,2047,0.0358240008354187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,127,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,4095,0.0459146648645401
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,8191,0.07682133217652638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,8191,0.07253333429495494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,16383,0.13587733109792074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,16383,0.11981866757074992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,4095,0.04800533254941305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,32767,0.2548853357632955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1,0.008837333569924036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,3,0.009045333291093508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,7,0.008837333569924036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,31,0.009589333087205887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,63,0.009701333319147428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1,0.009178666397929192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,127,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,255,0.01470400020480156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,511,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,511,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1023,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1023,0.02749866743882497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,32767,0.21799467007319132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,2047,0.03549866626660029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,4095,0.048154667019844055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,4095,0.047728002071380615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,8191,0.07746666669845581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,8191,0.072543998559316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,16383,0.13662933309872946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,16383,0.11975466211636861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,32767,0.25332266092300415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,2047,0.03366933266321818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,7,0.011701333026091257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,15,0.011653333902359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,31,0.011765333513418833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,63,0.012272000312805176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,127,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,255,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1023,0.012165332833925882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,2047,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,2047,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,4095,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,4095,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,8191,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,8191,0.01758933315674464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,16383,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,16383,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,32767,0.022986667851607006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,32767,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,15,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,511,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1023,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1023,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,2047,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,2047,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,32767,0.2177600065867106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,4095,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,63,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,8191,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,8191,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,16383,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,16383,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,32767,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,32767,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,3,0.008767999708652496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,7,0.008890666688481966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,15,0.008853333070874214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1,0.008703999842206636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,63,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,127,0.008976000050703684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,127,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,511,0.023141334454218548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,511,0.02699733277161916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1023,0.02959999938805898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1023,0.03133866687615713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,2047,0.03979199876387914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,2047,0.039893334110577904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,4095,0.06239999830722809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,4095,0.05832533538341522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,8191,0.10760533809661865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,8191,0.09518399834632874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,16383,0.19608000914255777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,16383,0.16905067364374796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,4095,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,32767,0.3729333480199178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1,0.009285333255926767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,7,0.009290666629870733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,15,0.009183999771873156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,31,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,63,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,127,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,255,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,255,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,511,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,511,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1023,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1023,0.0316746657093366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,2047,0.04188266893227895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,2047,0.04052799940109253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,4095,0.06434666613737743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,4095,0.06025066475073496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,8191,0.10785067081451416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,8191,0.09700799981753032
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,16383,0.19611199696858725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,32767,0.3734026749928792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,255,0.021695998807748158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,32767,0.3148426612218221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1,0.011989332735538483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,3,0.012202666451533636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,7,0.011855999628702799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,16383,0.17068266868591309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,32767,0.31804800033569336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,511,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,511,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,15,0.011610666910807291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,4095,0.017594666530688603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,4095,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,8191,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,16383,0.021642667551835377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,16383,0.019930666933457058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,32767,0.023418667415777843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,32767,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,3,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,31,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,31,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1023,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1023,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,2047,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,4095,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,4095,0.015461333096027374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,8191,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,8191,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,8191,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,16383,0.023530667026837666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,16383,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,32767,0.037791999677817024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,32767,0.024143998821576435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1,0.045791998505592346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1,0.03791466603676478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,3,0.0461760014295578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,3,0.03998400022586187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,7,0.04764799773693085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,7,0.04194133480389913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,15,0.05618133147557577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,15,0.05004266897837321
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,31,0.05593599875768026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,31,0.050101334849993386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,63,0.056421334544817604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,63,0.05023466547330221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,127,0.06638933221499126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,127,0.06053866446018219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,255,0.09743466973304749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,255,0.09121066331863403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,511,0.1622986694176992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,511,0.15254400173823038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,2047,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1023,0.2755146622657776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,2047,0.5399893522262573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,2047,0.52129065990448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,4095,1.0565653642018635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1,0.04384533564249674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1,0.040149333576361336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,3,0.04589866598447164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,3,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,7,0.04594666759173075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,7,0.040207999447981514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,15,0.05583466589450836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,15,0.050000001986821495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,4095,1.0029760201772053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,31,0.05612266560395559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,2047,0.014783999572197596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,63,0.056143999099731445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1023,0.2876586715380351
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,127,0.06620266536871593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,127,0.062122667829195656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,255,0.09716266393661499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,255,0.09308266639709473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,511,0.15927466750144958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,511,0.15466666221618652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1023,0.28748265902201336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1023,0.27557865778605145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,2047,0.5458933512369791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,31,0.05003199974695841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,63,0.05035200218359629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,4095,1.0189226468404133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,3,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,3,0.013823999712864557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,7,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,7,0.013536000003417334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,15,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,15,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,31,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,31,0.013962666193644205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,63,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,2047,0.5236746470133463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,127,0.013834666460752487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,127,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,255,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,255,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,511,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,4095,1.0684320131937664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1023,0.020986666282018025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1023,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,2047,0.040362666050593056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,2047,0.02719466636578242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,4095,0.06204266846179962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,63,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,511,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,3,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,7,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,7,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,15,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,15,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,31,0.016719999412695568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,31,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,63,0.01552533358335495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,63,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,127,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,127,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,255,0.015562667200962702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,511,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,511,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1023,0.03506666670242945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,3,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,2047,0.06017066538333893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,2047,0.03961600114901861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,255,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,4095,0.1016480028629303
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,4095,0.04366933306058248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,4095,0.062261333068211876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,3,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,7,0.013365333278973898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,15,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,31,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,31,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,63,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,127,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,127,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,255,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,255,0.014570667097965876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,511,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,511,0.019866666446129482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1023,0.03164266546567281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1023,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,2047,0.051967998345692955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,2047,0.047882666190465294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,4095,0.09097066521644592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,4095,0.0869760016600291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,8191,0.1709973414738973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,8191,0.1625546713670095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,3,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,16383,0.31306666135787964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1023,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,32767,0.6457706689834595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,32767,0.702613353729248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,3,0.010373333469033241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,7,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,15,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,16383,0.32862399021784466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,31,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,31,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,63,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1,0.009093333035707474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,3,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,127,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,255,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,255,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,511,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1023,0.033413333197434746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1023,0.029845332105954487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,2047,0.052229334910710655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,63,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,2047,0.04822933177153269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,4095,0.09110400080680847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,4095,0.08674666285514832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,8191,0.17099199692408243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,8191,0.16275200247764587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,16383,0.3284426728884379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,16383,0.3131680091222127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,32767,0.6535840034484863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,32767,0.703167994817098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,3,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,511,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,511,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,511,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,127,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,63,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,2047,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,2047,0.01579733317097028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,4095,0.01921066641807556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,4095,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,8191,0.021295999487241108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,8191,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,16383,0.02532266577084859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,32767,0.042208001017570496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1023,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,32767,0.027477333943049114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,7,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,16383,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,255,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,15,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1023,0.011621333658695221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,2047,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,2047,0.01598400001724561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,4095,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,4095,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,8191,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,8191,0.0195573332409064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,16383,0.0383840004603068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,16383,0.024005333582560223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,32767,0.05797866483529409
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1,0.08483733733495076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1,0.07061333457628886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,3,0.08688533306121826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,3,0.0729066679875056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,7,0.08869866530100505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,7,0.07660800218582153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,15,0.1055573324362437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,15,0.09425066908200581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,31,0.10570666193962097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,31,0.09502399961153667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,63,0.10738133390744527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,63,0.09544000029563904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,32767,0.03754666695992152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,127,0.11544000109036763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,255,0.1871253252029419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,511,0.3075093428293864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,127,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,511,0.2964586615562439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1023,0.5624853372573853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1023,0.5372159878412882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,127,0.12575999895731607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1,0.08500267068545024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,255,0.17518933614095053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1,0.07049066821734111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,3,0.08656000097592671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,3,0.07272533575693767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,7,0.08803199728329976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,7,0.07671999931335449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,15,0.1055626670519511
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,15,0.09489066402117412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,31,0.10553066929181416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,31,0.09501333038012187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,63,0.10693867007891338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,63,0.09551466504732768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,127,0.1255626678466797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,127,0.11549866199493408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,255,0.18715733289718628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,2047,1.0208053588867188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,511,0.31083200375239056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,511,0.2969013253847758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1023,0.5705120166142782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1023,0.5375253359476725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,2047,1.0721546808878581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,2047,1.0874773661295574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,2047,1.0387252966562908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,3,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,3,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,7,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,7,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,15,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,31,0.01754666616519292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,31,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,63,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,255,0.17697600523630777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,63,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,127,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,15,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,127,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,255,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,255,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,511,0.02142400046189626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,511,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1023,0.03757333258787791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1023,0.025674665967623394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,2047,0.0622026671965917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,2047,0.04402133325735728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1,0.021194666624069214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1,0.019551999866962433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,3,0.021274665991465252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,3,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,7,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,7,0.019482667247454327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,15,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,15,0.01882133384545644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,31,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,31,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,63,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,127,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,127,0.018826667219400406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,255,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,255,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,511,0.023141334454218548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1023,0.05490666627883911
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1023,0.03579200059175491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,2047,0.09716799855232239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,63,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,1,0.13385599851608276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,511,0.03460799902677536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,3,0.14014933506647745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,7,0.16881599028905234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,2047,0.05824000140031179
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,7,0.14646400014559427
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,1,0.16103466351826987
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,15,0.18169599771499634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,31,0.20374933878580728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,31,0.18319465716679892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,63,0.20573866367340088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,63,0.1839359998703003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,127,0.2427146633466085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,127,0.22311999400456747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,255,0.36585064729054767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,15,0.20385066668192545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,511,0.6129066546758016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,3,0.16498667001724243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,1,0.16076800227165222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,1,0.13398399949073792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,3,0.16555733482042947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,3,0.13802666465441385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,7,0.16880534092585245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,7,0.14469333489735922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,255,0.34515734513600665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,15,0.20355733235677084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,31,0.2060533364613851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,31,0.18145600954691568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,511,0.5847733418146769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,63,0.20567999283472696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,127,0.24278932809829712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,127,0.2229493260383606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,255,0.36583467324574787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,255,0.3452906608581543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,15,0.18121600151062012
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,511,0.6281386613845825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,511,0.5851360162099203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,1,0.021690666675567627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,3,0.025301332275072735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,3,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,7,0.02699200063943863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,7,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,15,0.025813333690166473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,15,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,31,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,31,0.02145066608985265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,63,0.025514667232831318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,63,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,127,0.025583999852339428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,127,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,63,0.18319465716679892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,255,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,255,0.02143999934196472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,511,0.03774933268626531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,511,0.029493334392706554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,1,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,1,0.02920000006755193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,3,0.031685332457224526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,3,0.02922666569550832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,7,0.03165333221356074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,7,0.029546665648619335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,15,0.031717332700888314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,15,0.029103999336560566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,31,0.03173333406448364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,31,0.02917333443959554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,63,0.03182933231194814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,63,0.0286613330245018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,127,0.03159466634194056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,127,0.02770666778087616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,255,0.03714666763941447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,255,0.027765333652496338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,511,0.05794133245944977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,511,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,1,0.3141706585884094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,1,0.025055999557177227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,1,0.25918400287628174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,3,0.32468799750010174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,3,0.26948267221450806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,7,0.33085866769154865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,7,0.2837173342704773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,15,0.400383989016215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,15,0.3553546667098999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,31,0.40068801244099933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,31,0.35540799299875897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,63,0.4036480188369751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,63,0.3577599922815959
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,127,0.47631998856862384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,127,0.43534934520721436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,1,0.3124693234761556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,255,0.7229066689809164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,255,0.6784959634145101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,3,0.3246399958928426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,3,0.29175466299057007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,7,0.331061323483785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,7,0.28338666756947833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,15,0.40053868293762207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,15,0.35555732250213623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,31,0.40030932426452637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,31,0.3720373312632243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,1,0.269375999768575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,63,0.41733864943186444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,63,0.3580640157063802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,127,0.4762880007425944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,127,0.4376266797383626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,1,0.0395359992980957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,255,0.7513919671376547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,3,0.04001066585381826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,3,0.035375999907652535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,7,0.03974399964014689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,7,0.035445332527160645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,15,0.039749334255854286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,15,0.03473600000143051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,31,0.039962666730086006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,31,0.033610666791598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,255,0.6791306336720785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,63,0.039850667119026184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,63,0.03422400106986364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,127,0.04161600023508072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,127,0.0356480007370313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,255,0.04266666869322459
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,255,0.03496533383925756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,1,0.05485333502292633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,1,0.047968000173568726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,3,0.054133335749308266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,3,0.0458133320013682
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,7,0.05513600011666616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,7,0.04602666695912679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,1,0.03543466577927271
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,15,0.04747733473777771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,31,0.054586668809254967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,31,0.04752533137798309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,63,0.056314667065938316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,63,0.04704533517360687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,127,0.05691199998060862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,127,0.047040000557899475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,255,0.04990933338801066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,3,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,3,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,15,0.05417599777380625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,7,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,15,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,15,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,31,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,31,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,63,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,255,0.062090665102005005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,127,0.019461333751678467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,255,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,7,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,511,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,511,0.03235200047492981
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1023,0.05180799961090088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1023,0.05203733344872793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,2047,0.09108266234397888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,63,0.019007999449968338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,127,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,4095,0.17110933860143027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,4095,0.16274133324623108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,255,0.02332266668478648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,8191,0.3102826674779256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,16383,0.6608320077260336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,16383,0.6030986706415812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,2047,0.0888320008913676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,8191,0.33079467217127484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,32767,1.3895947138468425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,32767,1.8736426035563152
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,3,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,3,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,7,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,15,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,31,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,31,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,63,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,63,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,127,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,127,0.019258666783571243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,255,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,255,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,511,0.03127466638882955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,511,0.03373866776625315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1023,0.05078400174776713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1023,0.05193600058555603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,2047,0.09089066584904988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,2047,0.08884800473848979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,4095,0.17307732502619425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,4095,0.16266666849454245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,8191,0.33504533767700195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,8191,0.30883200963338214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,7,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,16383,0.6723466714223226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,3,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,32767,1.5386187235514324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,7,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,255,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,511,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,511,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1023,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1023,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,2047,0.01597333326935768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,2047,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,32767,2.3827786445617676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,4095,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,8191,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,8191,0.020373333245515823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,16383,0.03923733284076055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,16383,0.6024426619211832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,32767,0.059061333537101746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,32767,0.04364799956480662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,7,0.011893333246310553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,31,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,63,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,255,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,511,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,511,0.012122667084137598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,2047,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,2047,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,4095,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,4095,0.018906666586796444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,8191,0.03818666686614355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,8191,0.024138666689395905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,16383,0.05940799911816915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,16383,0.025754667818546295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,32767,0.09981866677602132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,32767,0.05810666580994924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,4095,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,1,0.6188533306121826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,1,0.5130346616109213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,16383,0.03841066608826319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,3,0.6420853137969971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,3,0.5338240067164103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,7,0.6564533313115438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,7,0.5599733193715414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,15,0.7948160171508789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,31,0.7936800320943197
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,31,0.7057546774546305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,63,0.8415199915568033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,63,0.7082453568776449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,15,0.7038719654083252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,127,0.95087997118632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,127,0.8617599805196127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,1,0.6178239981333414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,1,0.5314186811447144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,3,0.6437280178070068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,3,0.5332746505737305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,7,0.6545386711756388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,7,0.5598080158233643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,15,0.7947946389516195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,15,0.7024959723154703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,31,0.7051839828491211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,31,0.7964906692504883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,63,0.8018080393473307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,63,0.7091573079427084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,1,0.07054399947325389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,127,0.9901066621144613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,127,0.8681173324584961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,3,0.07028266787528992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,3,0.06006933252016703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,7,0.07076266904671986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,7,0.058415999015172325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,15,0.07002133131027222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,15,0.058287998040517174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,31,0.07042133311430614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,1,0.058687999844551086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,31,0.05844266712665558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,63,0.07062399884064992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,63,0.0582239975531896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,127,0.07071466743946075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,127,0.05965333183606466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,1,0.10107733805974324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,1,0.08561600248018901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,3,0.09986666838328044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,3,0.08528533577919006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,7,0.10057600339253743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,15,0.10013866424560547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,15,0.08506666620572408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,31,0.0999679962793986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,31,0.0851039985815684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,63,0.09921066959698994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,63,0.08602133393287659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,127,0.09966400265693665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,127,0.08458667000134786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,7,0.08475733796755473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,1,1.2347573439280193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,3,1.2820586363474529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,7,1.2777547041575115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,1,1.012997309366862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,7,1.118618647257487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,3,1.0611733595530193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,15,1.5551466941833496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,15,1.3998719851175945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,31,1.6024746894836426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,31,1.4173386891682942
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,63,1.4354186058044434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,1,1.2275573412577312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,3,1.0653653144836426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,63,1.6405119895935059
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,1,1.020949363708496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,7,1.3246453603108723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,7,1.0963520208994548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,15,1.6109174092610676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,15,1.4092159271240234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,31,1.6327679951985676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,31,1.437621275583903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,1,0.1272479991118113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,63,1.6546826362609863
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,63,1.4739519755045574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,1,0.10753066341082256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,3,0.1260640025138855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,3,0.10610666871070862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,7,0.12708266576131186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,3,1.2926613489786785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,7,0.105813334385554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,15,0.12688533465067545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,15,0.10693333546320598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,31,0.12617066502571106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,31,0.10642666618029277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,63,0.12702932953834534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,63,0.10592533151308696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,1,0.18582399686177573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,1,0.15959999958674112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,3,0.1850879987080892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,3,0.1588266690572103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,7,0.18589866161346436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,15,0.18544000387191772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,15,0.15822399655977884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,31,0.18607999881108603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,31,0.15893866618474325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,3,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,3,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,7,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,7,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,15,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,63,0.18538665771484375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,63,0.15961066881815592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,31,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,31,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,63,0.01916266605257988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,63,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,127,0.021397332350413006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,127,0.019445333629846573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,255,0.029461334149042766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,255,0.02924799919128418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,511,0.0480373352766037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,511,0.04573333263397217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1023,0.08096000055472057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1023,0.0788213312625885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,2047,0.14653333028157553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,4095,0.28357332944869995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,7,0.15981866916020712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,4095,0.2794613242149353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,8191,0.5468213160832723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,2047,0.14964266618092856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,16383,1.1761546929677327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,3,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,3,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,8191,0.5553973515828451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,7,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,7,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,15,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,31,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,31,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,63,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,15,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,127,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,16383,1.3039626280466716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,255,0.031471999982992806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,255,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,15,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,511,0.04795733094215393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,511,0.04608533283074697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1023,0.08143466711044312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,2047,0.14878400166829428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,63,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,4095,0.2853333353996277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,4095,0.27932266394297284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,8191,0.5583253304163615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,8191,0.5474986632664999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,127,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,16383,1.3679146766662598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,16383,1.3593920071919758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,2047,0.14686933159828186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,511,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,511,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1023,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1023,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,2047,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,3,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,4095,0.02312533309062322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,4095,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,8191,0.04001600046952566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,8191,0.027317332724730175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1023,0.07863466441631317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,16383,0.060229331254959106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,3,0.012170666207869848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,7,0.011733333269755045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,31,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,31,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,127,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,511,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,511,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,16383,0.04231466849644979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1023,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,2047,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,2047,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,4095,0.0394400010506312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,4095,0.023605334262053173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,8191,0.059088001648585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,8191,0.03775999943415324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1023,0.01434133326013883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,2047,0.02096533278624217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,16383,0.05758399764696757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1,0.015360000232855478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,3,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,3,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,3,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,16383,0.10035199920336406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,3,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,7,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,7,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,15,0.019541333119074505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,15,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,15,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,31,0.019717333217461903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,31,0.019509332875410717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,31,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,31,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,7,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,63,0.01964266722400983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,63,0.017509333789348602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,63,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,127,0.02092266579469045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,127,0.019440000255902607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,127,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,127,0.017397332936525345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,255,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,255,0.023344000180562336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,255,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,15,0.01740266631046931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,511,0.019567999988794327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,511,0.03143466760714849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,511,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,511,0.029285334050655365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1023,0.019146667172511418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,7,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1023,0.04769066472848257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,63,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1023,0.04578666885693868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,2047,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,2047,0.08089600006739299
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,2047,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,2047,0.08064533273379008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,4095,0.019578666736682255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,4095,0.15028267105420431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,4095,0.14844800035158792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,8191,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1023,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,8191,0.28645867109298706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,255,0.020917333662509918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,8191,0.2837546666463216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,16383,0.019839999576409657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,4095,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,16383,0.01811733345190684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,16383,0.5974773168563843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,8191,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,3,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,7,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,7,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,31,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,31,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,255,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,511,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,511,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,16383,0.5995946725209554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,2047,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,2047,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,2047,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1023,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1023,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,4095,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,4095,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,4095,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,8191,0.02731200059254964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,8191,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,8191,0.025061334172884624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,2047,0.016757333030303318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,16383,0.042538667718569435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,16383,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,16383,0.03545066714286804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,3,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,16383,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,127,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,255,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,255,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,511,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,511,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,511,0.0122079998254776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1023,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1023,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,2047,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,2047,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,2047,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,2047,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,4095,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,4095,0.02298133323589961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,4095,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,4095,0.02109866589307785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,8191,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,8191,0.03782933453718821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,8191,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,8191,0.03025600065787633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,16383,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,16383,0.05890133480230967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,16383,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1,0.011727999895811081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,16383,0.05198933184146881
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,3,0.011877333124478659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,3,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,3,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,3,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,7,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,7,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,7,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,15,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,15,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,15,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,31,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,31,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,31,0.012645332763592402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,31,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,63,0.012047999848922094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,63,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,63,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,63,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,127,0.011770666887362799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,127,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,127,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,255,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,255,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,255,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,255,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,511,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,511,0.01431999976436297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,511,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,511,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1023,0.012096000214417776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1023,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1023,0.012234666695197424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1023,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,2047,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,2047,0.023978665471076965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,2047,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,2047,0.022885332504908245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,4095,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,4095,0.03878399978081385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,4095,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,4095,0.03148266673088074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,8191,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1,0.011616000284751257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,8191,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,8191,0.05425600210825602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,16383,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,16383,0.10288533568382263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1,0.009482666850090027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1,0.009103999783595404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,8191,0.06038933495680491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,3,0.009296000003814697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,3,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,7,0.009061333412925402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,7,0.00949866697192192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,16383,0.08815999825795491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,15,0.009072000160813332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,15,0.009045333291093508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,3,0.009237333511312803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,15,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,31,0.009392000113924345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,31,0.009066666786869368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,31,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,63,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,127,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,127,0.009072000160813332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,127,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,127,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,7,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,255,0.009189333145817121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,255,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,255,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,511,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,511,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1023,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1023,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1023,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,2047,0.018826667219400406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,2047,0.021269333859284718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,2047,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,2047,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,4095,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,4095,0.027210667729377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,4095,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,4095,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,8191,0.018042666216691334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,8191,0.036159999668598175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,8191,0.021029333273569744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,8191,0.037647999823093414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,16383,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,16383,0.053039997816085815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,16383,0.021242665747801464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,16383,0.05409599840641022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,32767,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,32767,0.08683733145395915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,32767,0.020975999534130096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,32767,0.08163733283678691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,65535,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,65535,0.15863999724388123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,65535,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,16383,0.01563199982047081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,65535,0.14030933380126953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,31,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,63,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,127,0.011616000284751257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,127,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,255,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,511,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1023,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,255,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,2047,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,2047,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,511,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,2047,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,4095,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,4095,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1023,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,4095,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,8191,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,8191,0.015450666348139444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,8191,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,8191,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,16383,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,16383,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,4095,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,16383,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,16383,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,32767,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,32767,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,32767,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,32767,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,65535,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,65535,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,3,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,3,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,65535,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,65535,0.022064000368118286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,15,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,2047,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,31,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,63,0.01051733394463857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,255,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,31,0.010581333190202713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,511,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,511,0.012133333832025528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1023,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1023,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1023,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,2047,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,2047,0.011621333658695221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,2047,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,2047,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,4095,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,4095,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,8191,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,8191,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,8191,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,8191,0.016629333297411602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,16383,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,16383,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,16383,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,16383,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,32767,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,32767,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,32767,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,32767,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,65535,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,65535,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,65535,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,65535,0.02091199904680252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,31,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,127,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,255,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,511,0.011770666887362799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1023,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1023,0.012005332857370377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,2047,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,2047,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,511,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,4095,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1023,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,4095,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,4095,0.015568000574906668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,8191,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,511,0.012448000411192576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,8191,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,2047,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,8191,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,16383,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,4095,0.014298666268587112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,16383,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,16383,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,32767,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,32767,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,8191,0.015615999698638916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,32767,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,16383,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,65535,0.023317334552605946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,65535,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,65535,0.022645334402720135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1,0.00892800030608972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1,0.009279999881982803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,3,0.008896000062425932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,3,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,7,0.008832000195980072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,65535,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,7,0.008682666967312494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,15,0.009429333110650381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,15,0.009290666629870733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,32767,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,3,0.010570666442314783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,31,0.009306666751702627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,31,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,63,0.009136000027259191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,127,0.009509333098928133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,127,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,127,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,255,0.00916800027092298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,255,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,255,0.013967999567588171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,31,0.009045333291093508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,511,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,511,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,255,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1023,0.021194666624069214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1023,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1023,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,2047,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,2047,0.025087999800841015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,127,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,2047,0.029258665939172108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,4095,0.0335413341720899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,4095,0.02164799968401591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1023,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,2047,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,8191,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,8191,0.0480373352766037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,8191,0.04800533254941305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,511,0.013744000345468521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,16383,0.017653333644072216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,16383,0.07755733529726665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,16383,0.07282133400440216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,4095,0.03572266548871994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,32767,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,8191,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,32767,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,32767,0.12171733379364014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,16383,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,65535,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,65535,0.2511253356933594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,65535,0.021498667697111767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,65535,0.21990400552749634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,32767,0.13622933626174927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,3,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,7,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,63,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,255,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,255,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,511,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,511,0.012618667135636011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,511,0.01190399999419848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1023,0.011760000139474869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,2047,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,2047,0.01184533288081487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,2047,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,4095,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,4095,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,4095,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1023,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,8191,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,8191,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,8191,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,8191,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,16383,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,16383,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,16383,0.012730666746695837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,2047,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,32767,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,32767,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,32767,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,4095,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,16383,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,65535,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,65535,0.023285334308942158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,65535,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,65535,0.022917332748572033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,3,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,15,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,31,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,31,0.01163200040658315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,32767,0.021002667645613354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,15,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,127,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,255,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,255,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,511,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,511,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,511,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1023,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,31,0.011722666521867117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1023,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,2047,0.011674666156371435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,2047,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,2047,0.013424000392357508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,4095,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1023,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,4095,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,4095,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,8191,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,8191,0.015872000406185787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,8191,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,8191,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,16383,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,16383,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,16383,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,16383,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,4095,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,32767,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,32767,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,32767,0.019498666127522785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,2047,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,65535,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,65535,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,65535,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,32767,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,15,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,15,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,15,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,31,0.01051733394463857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,31,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,63,0.010506667196750641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,65535,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,127,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,255,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,511,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,511,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,511,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1023,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1023,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,2047,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,2047,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,2047,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,2047,0.013823999712864557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,4095,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,4095,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,4095,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,8191,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,8191,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,8191,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,8191,0.017103999853134155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,16383,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,16383,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,16383,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,16383,0.018543999642133713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,32767,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,32767,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,4095,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,32767,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,65535,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,65535,0.037461332976818085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,65535,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1,0.027456000447273254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1,0.027066667874654133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1,0.023205332458019257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,3,0.027098665634791057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,3,0.02718399961789449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,32767,0.011727999895811081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,3,0.023152001202106476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,3,0.023557332654794056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,7,0.027317332724730175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,7,0.0278613343834877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,7,0.023562667270501454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,15,0.027776000400384266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,15,0.027327999472618103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,15,0.02362666775782903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,15,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,31,0.03176533430814743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,31,0.0315786674618721
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,31,0.029552000264326733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,7,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,63,0.031680000325044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,63,0.03158933420976003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,63,0.02951466788848241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,63,0.02940266579389572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,127,0.03200533241033554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,127,0.03294399877389272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,127,0.03162133445342382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,127,0.029311999678611755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,31,0.029829333225886028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,255,0.03759466608365377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,255,0.02957333376010259
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,65535,0.029120000700155895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,511,0.05380799869696299
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,511,0.02943466603755951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,511,0.0499946673711141
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1023,0.03180799881617228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,255,0.03166399896144867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1023,0.08692266543706258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1023,0.08382399876912434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,255,0.03554133325815201
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,2047,0.03158933420976003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,511,0.032458665470282234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,2047,0.15272000432014465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,2047,0.15029333035151163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,4095,0.032831999162832894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,4095,0.2874186635017395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,4095,0.28361600637435913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,8191,0.03352533280849457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,2047,0.02924799919128418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,8191,0.5580586592356364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,8191,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,4095,0.029605334003766377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1,0.011519999553759893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,8191,0.5482293367385864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1023,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,63,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,127,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,255,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,255,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,511,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,511,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1023,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,2047,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,2047,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,2047,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,2047,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,4095,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,4095,0.027215999861558277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,4095,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,4095,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,8191,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,8191,0.04214933514595032
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,8191,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,8191,0.03440000116825104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,3,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,3,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,3,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1023,0.013797332843144735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,3,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,7,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,7,0.01157333329319954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,15,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,15,0.012698666503032049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,15,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,31,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,31,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,7,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,7,0.013434667140245438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,63,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,63,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,15,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,63,0.011930666863918304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,127,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,127,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,127,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,31,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,31,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,255,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,63,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,511,0.01463466634353002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,511,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,255,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1023,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1023,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,2047,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,2047,0.025029333929220837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,255,0.01231466606259346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,2047,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,4095,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,4095,0.0401706670721372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,4095,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,4095,0.03164266546567281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,8191,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1023,0.015978666643301647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,8191,0.061280002196629844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,8191,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,8191,0.05215999980767568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,2047,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1,0.015370666980743408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,3,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,3,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,3,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,3,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,7,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,7,0.014783999572197596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,7,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,15,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,15,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,15,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,15,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,31,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,31,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,31,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,63,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,63,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,63,0.01451733335852623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,127,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,127,0.014501333236694336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,127,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,127,0.014352000008026758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,255,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,255,0.015642666568358738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,511,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,511,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,511,0.013610667238632837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,511,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1023,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1023,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1023,0.013653332988421122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1023,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,7,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,2047,0.03978666663169861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,2047,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,2047,0.03146133323510488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,4095,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,4095,0.060090666015942894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,4095,0.016783999900023144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,4095,0.05323199927806854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,8191,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,8191,0.10335999727249146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,8191,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1,0.009103999783595404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1,0.009530666594703993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1,0.010522666076819101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,3,0.009189333145817121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,3,0.00902399979531765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,3,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,7,0.008805333326260248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,7,0.009152000149091085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,7,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,8191,0.08867733677228291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,15,0.008832000195980072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,255,0.014757333944241205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,31,0.009285333255926767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,31,0.009050666665037474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,2047,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,15,0.009125333279371262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,127,0.010501333822806677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,127,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,127,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,255,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,255,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,511,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,511,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,511,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1023,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1023,0.023562667270501454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1023,0.027722666660944622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,2047,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,2047,0.029893333713213604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,2047,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,2047,0.03164800008138021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,4095,0.019482667247454327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,4095,0.04178666571776072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,4095,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,4095,0.04159466673930486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,8191,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,8191,0.062368000547091164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,8191,0.02369066576162974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,8191,0.06065066655476888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1023,0.02310933421055476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,16383,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,16383,0.1074666678905487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,16383,0.023445333043734234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,16383,0.09735999504725139
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,32767,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,32767,0.1954560081164042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,32767,0.023589332898457844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,32767,0.17110933860143027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,65535,0.01947733387351036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,65535,0.3719253142674764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,65535,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,65535,0.3186826705932617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,3,0.011786667009194693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,7,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,31,0.010506667196750641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,31,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,511,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,511,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,3,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1023,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,2047,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,2047,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,2047,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,4095,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,4095,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,4095,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,4095,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,8191,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,8191,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,8191,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,8191,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,16383,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,16383,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,16383,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,32767,0.02213866760333379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,32767,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,32767,0.021338666478792827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,16383,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,65535,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,65535,0.0273333340883255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,3,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,3,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,7,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,7,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,31,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,31,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,65535,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,63,0.010357333347201347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,127,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,65535,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,255,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,255,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,255,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,511,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,511,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1023,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,2047,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,2047,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,4095,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,4095,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,8191,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,2047,0.013482666263977686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,511,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,8191,0.013482666263977686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,8191,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,16383,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,16383,0.021226666867733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,16383,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,16383,0.01966933285196622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1023,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,32767,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,32767,0.025498665869235992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,8191,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,32767,0.023168000082174938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,65535,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,65535,0.0394400010506312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,65535,0.03147733211517334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,63,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,255,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,511,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,511,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1023,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1023,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,32767,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,2047,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,2047,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,4095,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,4095,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1023,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,4095,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,8191,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,8191,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,8191,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,16383,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,4095,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,16383,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,2047,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,16383,0.02327466756105423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,32767,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,32767,0.03777066618204117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,8191,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,32767,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,32767,0.02977599948644638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,16383,0.02309866746266683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,65535,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,65535,0.05806399881839752
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1,0.04593066871166229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1,0.04578666885693868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1,0.039066667358080544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1,0.03749333322048187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,3,0.046207999189694725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,3,0.045925334095954895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,3,0.039520000418027244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,3,0.037802666425704956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,7,0.04821333289146423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,7,0.04621866842110952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,7,0.03961066653331121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,7,0.039520000418027244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,15,0.04892266790072123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,15,0.047770669062932335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,15,0.04155733436346054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,65535,0.052069331208864846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,31,0.05799466868241628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,31,0.056405335664749146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,63,0.05669866502285004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,63,0.05635199944178263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,63,0.050069332122802734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,63,0.050016000866889954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,127,0.05810666580994924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,15,0.041690667470296226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,127,0.050111999114354454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,127,0.05005866785844167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,255,0.05795200169086456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,255,0.06650666892528534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,255,0.05009600023428599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,255,0.06020799775918325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,127,0.05787200232346853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,511,0.09870400031407674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,31,0.05086933573087057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,511,0.09337600072224934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1023,0.059018666545550026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1023,0.16086933016777039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,511,0.058186665177345276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1023,0.052373334765434265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,31,0.05199466645717621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1023,0.15471466382344565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,2047,0.05813866853713989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,2047,0.05212266743183136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,2047,0.27992000182469684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,4095,0.058005332946777344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,4095,0.052245333790779114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,4095,0.5324533383051554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,2047,0.2898293336232503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,4095,0.5499893426895142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,3,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,3,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,3,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,7,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,7,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,7,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,7,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,15,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,15,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,15,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,15,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,31,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,31,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,31,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,31,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,63,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,63,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,63,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,511,0.05198400219281515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,127,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,127,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,127,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,255,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,255,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,255,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,255,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,511,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,511,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,511,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,511,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1023,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1023,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,2047,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,2047,0.027263998985290527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,2047,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,2047,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,4095,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,4095,0.0452159990866979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,4095,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,63,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,3,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,3,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,3,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,3,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1023,0.017103999853134155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,7,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,7,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,7,0.014789332946141561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,7,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,15,0.014837333311637243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,15,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,15,0.014682666709025701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,15,0.014303999642531076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,31,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,31,0.014655999839305878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,31,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,31,0.014303999642531076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,63,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,63,0.015568000574906668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,63,0.01403733342885971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,63,0.014720000326633453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,127,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,127,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,127,0.014271999398867289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,255,0.014101333916187286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,255,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,255,0.014325333138306936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,511,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,4095,0.03605866680542628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,127,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,511,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,511,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1023,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1023,0.02332266668478648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1023,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1023,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,2047,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,2047,0.04051200052102407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,2047,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,2047,0.03336533407370249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,511,0.017157333592573803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,4095,0.017477333545684814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,4095,0.06204266846179962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,255,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1,0.017642666896184284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,4095,0.054117331902186074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,3,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,3,0.018757333358128864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,3,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,3,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,7,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,7,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,7,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,7,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,15,0.017653333644072216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,15,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,4095,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,15,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1,0.01882133384545644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,31,0.01759999990463257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,31,0.018351999421914417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,31,0.01747200017174085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,63,0.01889066646496455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,63,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,63,0.017605333278576534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,63,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,127,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,127,0.01868266612291336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,127,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,127,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,255,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,15,0.01722666621208191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,255,0.01762666677435239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,31,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,511,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,511,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,511,0.017418666432301205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,511,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1023,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1023,0.03578133384386698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1023,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1023,0.02934933453798294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,2047,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,2047,0.05772800246874491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,255,0.017445333302021027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,2047,0.020986666282018025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,255,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,4095,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,4095,0.09537067015965779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,4095,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,4095,0.082805335521698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1,0.085125337044398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1,0.070592001080513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1,0.07045866549015045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,3,0.08585600058237712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,2047,0.05008533100287119
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,3,0.0865226686000824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,3,0.07026133437951405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,3,0.07040533423423767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1,0.08496000369389851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,7,0.08870933453241985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,7,0.07286933561166127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,7,0.07293866574764252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,15,0.09125333031018575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,15,0.09027199943860371
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,15,0.07640000184377034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,15,0.0765066643555959
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,31,0.10732266306877136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,31,0.10778666536013286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,31,0.09532266855239868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,31,0.09516266981760661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,63,0.10784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,63,0.10781332850456238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,63,0.09493866562843323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,127,0.10965333382288615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,127,0.1092639962832133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,127,0.09545600414276123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,127,0.09533333778381348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,255,0.10963732997576396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,255,0.12777599692344666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,255,0.09556800127029419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,7,0.08694400389989217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,255,0.11374400059382121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,511,0.10940266648928325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,511,0.1893813411394755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,511,0.17730667193730673
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1023,0.1095413366953532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1023,0.31411200761795044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1023,0.09539199868837993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1023,0.3031253417332967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,3,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,63,0.09541866183280945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,3,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,3,0.01553600033124288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,7,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,7,0.015333333363135656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,7,0.01544533297419548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,511,0.09528533617655437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,15,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,3,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,15,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,7,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,31,0.015605332950750986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,31,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,31,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,31,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,63,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,63,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,63,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,63,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,15,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,127,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,15,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,127,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,255,0.01685333376129468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,255,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,255,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,255,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,511,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,511,0.019391999890406925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,511,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,511,0.0189280000825723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,127,0.01568000018596649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1023,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,127,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1023,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1023,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1,0.01791999985774358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,3,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,3,0.01916266605257988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,3,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,3,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,7,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,7,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1023,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,7,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,15,0.01740266631046931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,15,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,15,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,31,0.01913600042462349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,31,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,31,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,31,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,63,0.018816000471512478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,63,0.019173332800467808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,7,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,15,0.018768000106016796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,127,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,127,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,127,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,127,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,63,0.018816000471512478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,255,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,63,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,255,0.01913600042462349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,255,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,511,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,511,0.023183998962243397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,511,0.018885333091020584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,511,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1023,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1023,0.035786665976047516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1023,0.01874133323629697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,255,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1023,0.030165334542592365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1,0.025589334468046825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,3,0.025914666553338368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,3,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,3,0.025114665428797405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,7,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,7,0.025583999852339428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,7,0.02347733328739802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,15,0.025125332176685333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,15,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,15,0.02349333216746648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,15,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,31,0.025242666403452556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,31,0.025674665967623394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,31,0.025146665672461193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,31,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,63,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,63,0.02632533262173335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,63,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,63,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,127,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,127,0.025445332129796345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,127,0.025237334271272022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,127,0.02508266766866048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,3,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,255,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,255,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,255,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,511,0.02718399961789449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,511,0.03789866715669632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,511,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1023,0.027280000348885853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1023,0.056287998954455055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,255,0.027290667096773785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1023,0.05060266455014547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,1,0.16286933422088623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,1,0.1638879974683126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,1,0.1340053379535675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,1,0.13383466998736063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,3,0.16296533743540445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,3,0.16291200121243796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1023,0.02513066679239273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,3,0.13337600231170654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,7,0.16885334253311157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,7,0.16874667008717856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,7,0.13831999897956848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,511,0.03336533407370249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,15,0.17283733685811362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,15,0.1728480060895284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,3,0.13300800323486328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,15,0.14428266882896423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,31,0.20775999625523886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,31,0.2076746622721354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,31,0.1812373399734497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,7,0.13828266660372415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,31,0.1819253365198771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,63,0.20779200394948324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,63,0.20821332931518555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,15,0.14441066980361938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,63,0.18288532892862955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,127,0.21000534296035767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,127,0.2103253404299418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,127,0.1832266648610433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,127,0.18321067094802856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,255,0.2119413415590922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,255,0.24709866444269815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,255,0.1832639972368876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,255,0.22234133879343668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,63,0.18311466773351034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,511,0.36951998869578045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,511,0.18458133935928345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,511,0.3449653387069702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,1,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,1,0.021317332983016968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,1,0.01941866676012675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,1,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,3,0.021381333470344543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,3,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,3,0.019440000255902607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,511,0.21046932538350424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,3,0.019493332753578823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,7,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,7,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,7,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,15,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,15,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,15,0.019461333751678467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,15,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,31,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,31,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,31,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,31,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,63,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,63,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,63,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,63,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,127,0.02120000123977661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,127,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,127,0.019541333119074505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,7,0.020010666300853092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,255,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,255,0.02145066608985265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,255,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,511,0.022831998765468597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,511,0.02532266577084859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,511,0.021002667645613354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,511,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,127,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,1,0.02752533306678136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,1,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,1,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,1,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,3,0.02716800073782603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,3,0.025685332715511322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,3,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,7,0.02775466690460841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,7,0.02693866689999898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,7,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,7,0.025498665869235992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,255,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,15,0.027237333357334137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,3,0.027600000301996868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,15,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,15,0.025381334125995636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,31,0.027141332626342773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,31,0.027429332335789997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,31,0.02537599951028824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,31,0.02515733242034912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,63,0.027237333357334137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,63,0.027322667340437572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,63,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,63,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,127,0.02716800073782603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,127,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,15,0.02658133457104365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,127,0.025706666211287182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,127,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,255,0.02741866558790207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,255,0.025237334271272022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,255,0.025519999365011852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,255,0.025754667818546295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,511,0.027285332481066387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,511,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,511,0.02569066733121872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,511,0.03346133232116699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,1,0.04164266586303711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,1,0.04153066625197729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,1,0.037808001041412354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,1,0.03787733366092046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,3,0.04165866722663244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,3,0.04161066561937332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,3,0.03941866755485535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,3,0.037615999579429626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,7,0.04161600023508072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,7,0.04192000130812327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,7,0.03941866755485535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,15,0.04178666571776072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,15,0.04163199911514918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,15,0.03956266740957896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,15,0.03749866783618927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,31,0.041562666495641075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,31,0.04176533222198486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,31,0.03812266637881597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,31,0.03750933210055033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,63,0.042080000042915344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,63,0.04215999941031138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,63,0.03782933453718821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,7,0.03908800085385641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,63,0.039647998909155525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,127,0.04177066683769226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,127,0.0395413339138031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,127,0.03882133215665817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,127,0.038693333665529885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,255,0.0439573327700297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,255,0.04293333490689596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,255,0.03963200002908707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,255,0.03855466594298681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,511,0.042863999803860985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,511,0.062133332093556724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,511,0.03977066775163015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1,0.01421333352724711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1,0.009125333279371262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,3,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,3,0.014250667144854864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,3,0.009546666716535887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,7,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,7,0.01414399966597557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,15,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,15,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,31,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,31,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,31,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,63,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,511,0.05602666735649109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,63,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,63,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,127,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,255,0.01543466622630755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,255,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,511,0.01533866673707962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,511,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,511,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,127,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1023,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1023,0.02313599983851115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1023,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1023,0.01904533306757609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,2047,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,255,0.013914667069911957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,2047,0.03345600018898646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,2047,0.013679999858140945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,63,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,4095,0.015765332927306492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,4095,0.052255998055140175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,4095,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,4095,0.04841066896915436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,8191,0.01532799998919169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,8191,0.09292800227801006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,8191,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,8191,0.0869599978129069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,16383,0.015573333948850632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,16383,0.1711519956588745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,16383,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,16383,0.1649279991785685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,511,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,32767,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,32767,0.32948799928029376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,32767,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,32767,0.33318400382995605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,65535,0.01540800059835116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,2047,0.029205332199732464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,65535,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,65535,0.7359626293182373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,3,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,3,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,31,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,63,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,63,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,255,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,255,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,511,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,511,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,511,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,511,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1023,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1023,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1023,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,2047,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,2047,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,2047,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,4095,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,4095,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,4095,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,4095,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,8191,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,8191,0.0189280000825723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,8191,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,8191,0.017594666530688603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,16383,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,16383,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,16383,0.020986666282018025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,32767,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,32767,0.027386667827765148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,32767,0.013471999516089758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,32767,0.0249439999461174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,65535,0.04322666426499685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,65535,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,7,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,31,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,31,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,31,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,63,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,63,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,255,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,65535,0.7144266764322916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,511,0.012389333297808966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,511,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1023,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,65535,0.0334346666932106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1023,0.011973333855470022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,2047,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,2047,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,4095,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,4095,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,4095,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,4095,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,8191,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,8191,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,8191,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,8191,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,16383,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,16383,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,2047,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,16383,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,511,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,32767,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,32767,0.037834666669368744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,32767,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,32767,0.031370667119820915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,65535,0.06018133461475372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,16383,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,65535,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,7,0.010543999572594961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,15,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,15,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,31,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,255,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,511,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1023,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,2047,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,2047,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,2047,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,4095,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,4095,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,4095,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,65535,0.05220800141493479
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,8191,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,8191,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,8191,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,8191,0.021375998854637146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,16383,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,16383,0.037802666425704956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,16383,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,16383,0.030623999734719593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,32767,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,32767,0.059621334075927734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,32767,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,32767,0.05180799961090088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1023,0.013733333597580591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,65535,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,65535,0.1009333332379659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,65535,0.01340266689658165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,4095,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,65535,0.08699199557304382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,1,0.32052799065907794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,1,0.3204960028330485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,1,0.25969066222508747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,3,0.31859199206034344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,3,0.3202720085779826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,3,0.25964800516764325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,3,0.2590879996617635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,7,0.32895465691884357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,7,0.33002134164174396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,7,0.2678239941596985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,7,0.2675573428471883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,15,0.3368106683095296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,15,0.33558400472005206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,15,0.2814026673634847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,15,0.27956799666086835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,31,0.4081919987996419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,1,0.2592373291651408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,31,0.40824000040690106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,31,0.3561386664708455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,31,0.355461319287618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,63,0.4127946694691976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,63,0.410805344581604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,63,0.35837332407633465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,63,0.35733334223429364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,127,0.41463998953501385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,127,0.41437868277231854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,127,0.35930665334065753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,127,0.35970131556193036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,255,0.4166986544926961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,255,0.4864533344904582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,255,0.3595199982325236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,255,0.43538665771484375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,1,0.03127466638882955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,1,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,1,0.029839999973773956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,3,0.03166933357715607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,3,0.031354665756225586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,3,0.029264000554879505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,3,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,7,0.03190399954716364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,1,0.031557333966096245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,7,0.031136001149813335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,7,0.029845332105954487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,7,0.029733332494894665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,15,0.03181866556406021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,15,0.03153600047032038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,15,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,15,0.02959466725587845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,31,0.03164800008138021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,31,0.031658666829268135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,31,0.02945599953333537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,31,0.029893333713213604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,63,0.03143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,63,0.031658666829268135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,63,0.029882666965325672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,63,0.029674666623274486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,127,0.031354665756225586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,127,0.03138133386770884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,127,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,127,0.029450667401154835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,255,0.03390933324893316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,255,0.03155199935038885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,255,0.031744000812371574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,1,0.043866669138272606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,1,0.04399466514587402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,1,0.04161066561937332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,1,0.04163199911514918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,3,0.04398400088151296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,3,0.04368533194065094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,255,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,7,0.04381866753101349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,7,0.043621331453323364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,7,0.041722665230433144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,7,0.041877334316571556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,3,0.04178133110205332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,15,0.044069334864616394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,3,0.04162666698296865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,15,0.04375466704368591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,15,0.04206933577855428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,15,0.041690667470296226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,31,0.0444106658299764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,31,0.043791999419530235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,31,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,31,0.04158399999141693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,63,0.04423999786376953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,63,0.04364266494909922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,63,0.04180799921353658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,63,0.041706666350364685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,127,0.04378666480382284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,127,0.04371733466784159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,127,0.0421066681543986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,127,0.04161600023508072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,255,0.04615999758243561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,255,0.045909335215886436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,255,0.04207466542720795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,255,0.04177600145339966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,1,0.07285866638024648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,1,0.07283199826876323
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,1,0.0684799998998642
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,1,0.06849066913127899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,3,0.0738453318675359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,3,0.07400533556938171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,3,0.06834666430950165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,3,0.06832533578077953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,7,0.07412800192832947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,7,0.07348800202210744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,7,0.06834133466084798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,7,0.06851199766000111
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,15,0.07329066594441731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,15,0.07375999788443248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,15,0.06854400038719177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,15,0.06856533388296764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,31,0.07333866755167644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,31,0.07286400099595387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,31,0.06803733110427856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,31,0.06855466465155284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,63,0.07426133255163829
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,63,0.07371733089288075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,63,0.06840533514817555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,63,0.06819733480612437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,127,0.07471466561158498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,127,0.07425599793593089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,127,0.06844266752401988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,127,0.06663999954859416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,255,0.0788373351097107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,255,0.07635200023651123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,255,0.07039999961853027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,255,0.06850133339564006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,1,0.6323306560516357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,1,0.6323786576588949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,1,0.5113226572672526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,3,0.6325333515803019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,3,0.5117599964141846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,1,0.5111253261566162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,3,0.5117173194885254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,7,0.6505599816640218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,3,0.6324373483657837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,7,0.5315680106480917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,7,0.5291146834691366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,15,0.665120005607605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,7,0.6525813341140747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,15,0.6682453155517578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,15,0.5522186756134033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,15,0.5533279975255331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,31,0.8120480378468832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,31,0.8037599722544352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,31,0.706063985824585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,31,0.7054346402486166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,63,0.8139786720275879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,63,0.814250628153483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,63,0.709503968556722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,127,0.8208266894022623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,63,0.7082346280415853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,127,0.8203306992848715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,127,0.6928426424662272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,1,0.05247466762860616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,127,0.7120373249053955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,1,0.05216533442338308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,1,0.049925332268079124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,1,0.04994666576385498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,3,0.05211733281612396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,3,0.05203733344872793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,3,0.04981866478919983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,3,0.05026666820049286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,7,0.051914667089780174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,7,0.052341332038243614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,7,0.04995200037956238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,7,0.049925332268079124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,15,0.052042668064435325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,15,0.04961599906285604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,31,0.05197866757710775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,31,0.04806933303674062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,31,0.04824000100294749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,15,0.05232533315817515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,63,0.05184000233809153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,63,0.052154665191968284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,63,0.04798933366934458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,15,0.05017066498597463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,31,0.052341332038243614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,63,0.049679999550183616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,127,0.05203733344872793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,127,0.05232533315817515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,127,0.049098665515581764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,1,0.07855466504891713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,1,0.07897066573301952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,1,0.07252799967924754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,1,0.07258133093516032
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,127,0.04816000163555145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,3,0.07886399825414021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,3,0.07852800190448761
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,3,0.07452266911665599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,3,0.0729013333717982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,7,0.07895466685295105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,7,0.07468800246715546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,7,0.07267733414967854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,15,0.07866133252779643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,15,0.07866133252779643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,7,0.07870399951934814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,15,0.07284266750017802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,15,0.07285866638024648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,31,0.07881600161393483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,31,0.07926400005817413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,31,0.07246933380762736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,31,0.07268799841403961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,63,0.07860266665617625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,63,0.07870933413505554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,63,0.07449066638946533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,63,0.07427733143170674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,127,0.07867733140786488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,127,0.07984533409277599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,127,0.07462400197982788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,127,0.0759200006723404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,1,0.1389226714769999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,1,0.1400373379389445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,1,0.12641066312789917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,1,0.12654933333396912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,3,0.13916266957918802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,3,0.14040533701578775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,3,0.12598933776219687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,3,0.12774399916330972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,7,0.14012799660364786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,7,0.12601066629091898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,7,0.12618133425712585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,15,0.139082670211792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,15,0.14011733730634054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,15,0.12612266341845194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,15,0.12607999642690024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,31,0.13988266388575235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,31,0.13943466544151306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,31,0.1263040006160736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,31,0.12685333689053854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,7,0.1405333379904429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,63,0.1383840044339498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,63,0.12600533167521158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,63,0.1272586683432261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,63,0.13945600390434265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,127,0.13756799697875977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,127,0.127920001745224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,3,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,3,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,3,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,3,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,7,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,7,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,7,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,7,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,15,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,15,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,15,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,15,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,31,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,31,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,31,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,31,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,63,0.013546666751305262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,63,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,63,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,127,0.014394666999578476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,127,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,127,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,127,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,255,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,255,0.016832000265518825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,255,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,255,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,63,0.014346666634082794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,511,0.021456000705560047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,511,0.019802667200565338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,127,0.13833066821098328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,511,0.02363733450571696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1023,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1023,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1023,0.03332266708215078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,2047,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,127,0.12785067160924277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,2047,0.019541333119074505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,2047,0.05194133520126343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,4095,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1023,0.03217600037654241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,4095,0.09237333138783772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,4095,0.01959466685851415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,4095,0.08906666437784831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,8191,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,8191,0.1728000044822693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,8191,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,8191,0.16359999775886536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,16383,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,16383,0.3398826519648234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,16383,0.31250133117039997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,2047,0.05202133456865946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,16383,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,32767,0.9931519826253256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,32767,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,511,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,32767,0.6543146769205729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,7,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,32767,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,15,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,511,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,511,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,511,0.011898666620254517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1023,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,2047,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,2047,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,2047,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,4095,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,4095,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,4095,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,8191,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,8191,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,8191,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,8191,0.01950399950146675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,16383,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,16383,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,16383,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,16383,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,32767,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,32767,0.043375998735427856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,32767,0.03363733241955439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,3,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,7,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,15,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,31,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,31,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,127,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,255,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,32767,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,255,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,511,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,511,0.01163200040658315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1023,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1023,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1023,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,2047,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,2047,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,2047,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,2047,0.015573333948850632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,4095,0.019391999890406925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,4095,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,8191,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,8191,0.023183998962243397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,8191,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,8191,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,16383,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,16383,0.03774933268626531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,16383,0.013818666338920593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,16383,0.030053332448005676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,32767,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,32767,0.058917333682378135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,32767,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,32767,0.051776001850763954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,4095,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,7,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,15,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,511,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,511,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1023,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1023,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,2047,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,2047,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,2047,0.013557333499193192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,2047,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,4095,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,4095,0.023498666783173878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,4095,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,4095,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,8191,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,8191,0.039135999977588654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,8191,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,8191,0.031504000226656594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,16383,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,16383,0.05891199906667074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,16383,0.05208000044027964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,32767,0.013653332988421122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,32767,0.10127466917037964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,32767,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,3,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,3,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,16383,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,7,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,15,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,31,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,31,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,63,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,63,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,127,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,127,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,32767,0.0867199997107188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,255,0.029461334149042766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,7,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,511,0.04778666794300079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,511,0.04586666822433472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1023,0.08075733482837677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1023,0.07921599845091502
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,2047,0.1486293375492096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,2047,0.14648000399271646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,4095,0.2834239999453227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,4095,0.2794026732444763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,255,0.029493334392706554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,8191,0.547274669011434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,15,0.017157333592573803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,16383,1.3213813304901123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,16383,1.3597119649251301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,8191,0.5591733455657959
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,255,0.01258133351802826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,511,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,511,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1023,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,2047,0.018378666291634243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,4095,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,4095,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,8191,0.03995199998219808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,8191,0.02749866743882497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,16383,0.06006399790445963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,16383,0.04369066655635834
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,3,0.012096000214417776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,2047,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,511,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1023,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,2047,0.020799999435742695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,4095,0.03841066608826319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,4095,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,8191,0.058730666836102806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,8191,0.03760000069936117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,16383,0.09920533498128255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,16383,0.058005332946777344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1,0.012565333396196365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,3,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,7,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,7,0.012154666086037954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,15,0.012650666137536367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,31,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,31,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,63,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,63,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,127,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,127,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,255,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,255,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,511,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,511,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1023,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1023,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,2047,0.03975466638803482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,3,0.013904000322024027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,4095,0.061109334230422974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,2047,0.018837332725524902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,8191,0.10128000378608704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,8191,0.060421332716941833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,16383,0.1841920018196106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,16383,0.10001066327095032
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1,0.008805333326260248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,3,0.009114666531483332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,3,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,7,0.008714666590094566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,7,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,2047,0.025040000677108765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,15,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,31,0.00914666677514712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,4095,0.03987200061480204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,63,0.00919999989370505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,63,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,127,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,255,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,255,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,511,0.015466666469971338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,511,0.017680000513792038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1023,0.021840001145998638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,15,0.009205333267649015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,2047,0.027263998985290527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,2047,0.027263998985290527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,127,0.009194666519761086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,4095,0.03596800069014231
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,4095,0.037690666814645134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,8191,0.05297600229581197
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,8191,0.05183466772238413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,16383,0.08876267075538635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,16383,0.08084266881148021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,32767,0.15874133507410684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,32767,0.13827733198801676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1023,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,3,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,65535,0.25492266813913983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,7,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,31,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,65535,0.2999040087064107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,255,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,2047,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,2047,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,4095,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,8191,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,8191,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,16383,0.021375998854637146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,16383,0.021738665799299877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,32767,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,32767,0.02437866727511088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,65535,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,3,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,7,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,63,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,511,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1023,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,65535,0.02584533393383026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,4095,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,4095,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,8191,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,8191,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,15,0.011722666521867117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,16383,0.01945066700379054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,16383,0.019786667078733444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,2047,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,65535,0.025573333104451496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,65535,0.021733333667119343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,3,0.010384000216921171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,7,0.010527999450763067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,32767,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,127,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,127,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,255,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1023,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,2047,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,4095,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,4095,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,255,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,32767,0.019882666567961376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,16383,0.01961600035429001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,16383,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,2047,0.012666666259368261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,32767,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,32767,0.019391999890406925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,8191,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,65535,0.0374293327331543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,65535,0.02329600105683009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,3,0.009152000149091085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,7,0.009216000015536943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,15,0.009312000125646591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,63,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,127,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,255,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,255,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,511,0.021130666136741638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,511,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1023,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1023,0.02735466758410136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,2047,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,2047,0.035536001125971474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,4095,0.04794133206208547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,4095,0.04641066491603851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,8191,0.07670400043328603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,8191,0.07086933155854543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,16383,0.1361066699028015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1,0.009290666629870733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,16383,0.11997866630554199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,32767,0.25356799364089966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,32767,0.21777600049972534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,65535,0.4868160088857015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,65535,0.41225600242614746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,7,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,15,0.011946666985750198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,8191,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,255,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,511,0.011514666179815928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1023,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1023,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,2047,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,4095,0.014282666146755219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,4095,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,8191,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,511,0.012154666086037954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,16383,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,16383,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,32767,0.023317334552605946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,32767,0.021317332983016968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,65535,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,8191,0.019189332922299702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,7,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,15,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,127,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,3,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,511,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,2047,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,4095,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,4095,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,65535,0.023007998863856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,8191,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,8191,0.016869333883126576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,16383,0.018922666708628338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,16383,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,32767,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,32767,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,65535,0.038133333126703896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,15,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,31,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1023,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,2047,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,2047,0.014783999572197596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,4095,0.016672000288963318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,4095,0.016778666526079178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,8191,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,8191,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,16383,0.023077333966890972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,16383,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,65535,0.025205334027608235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,32767,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,65535,0.0569653312365214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,65535,0.03721066564321518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,3,0.027301333844661713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,32767,0.037621334195137024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,3,0.023418667415777843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,7,0.027258666853109997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,7,0.024133334557215374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,15,0.03146133323510488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,15,0.02942933390537898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,31,0.031290667752424874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,31,0.029653333127498627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,63,0.03152533372243246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,63,0.02977066735426585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,127,0.0373333344856898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,127,0.03339199970165888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,255,0.0539626677831014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,255,0.05236800014972687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,511,0.08457600076993306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1023,0.15230933825174967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1023,0.14814399679501852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,2047,0.2856160004933675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,2047,0.27662932872772217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,4095,0.5577386617660522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,4095,0.5332853396733602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,511,0.08679999907811482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,8191,1.1086400349934895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,3,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,15,0.011722666521867117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,15,0.011706666400035223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,31,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,31,0.01184533288081487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,63,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,127,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,127,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,255,0.011658667276302973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,511,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1023,0.014762666076421738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1023,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,2047,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,2047,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,4095,0.03884266565243403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,4095,0.0272533322374026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,8191,0.9965759913126627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,8191,0.042319998145103455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,3,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,3,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,7,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,7,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,15,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,15,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,31,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,31,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,63,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,63,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,127,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,127,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,255,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,255,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,511,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,511,0.014741333822409311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1023,0.019461333751678467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1023,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,2047,0.04049066702524821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,2047,0.025370667378107708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,4095,0.06052800019582113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,4095,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,8191,0.10148800412813823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,8191,0.06003733476003011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,8191,0.060165335734685264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,3,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,3,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,7,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,7,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,15,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,15,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,31,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,31,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,63,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,63,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,127,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,127,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,255,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,255,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,511,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,511,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1023,0.033802665770053864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1023,0.02145066608985265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,2047,0.06018666426340739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,2047,0.039605334401130676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,4095,0.10178666313489278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,8191,0.18475200732549033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1,0.009194666519761086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,8191,0.10139200091362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,3,0.008879999940594038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,7,0.009237333511312803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,15,0.00914666677514712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,63,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,127,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,255,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,255,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,4095,0.0610453337430954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,511,0.02722666660944621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1023,0.029520000020662945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1023,0.03126933425664902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,2047,0.04156800111134847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,63,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,2047,0.04004266609748205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,4095,0.06509333352247874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,8191,0.10845866799354553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,511,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,16383,0.19739200671513876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,16383,0.1688800056775411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,4095,0.058789332707722984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,32767,0.3173813422520955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,8191,0.09718400239944458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1,0.011551999797423681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,65535,0.7292799949645996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,3,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,65535,0.6086986859639486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,32767,0.37141867478688556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,63,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,511,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,511,0.01190399999419848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1023,0.012597333639860153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1023,0.011823999385039011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,2047,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,2047,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,7,0.011589333415031433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,4095,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,8191,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,8191,0.019013332823912304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,16383,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,16383,0.02120000123977661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,32767,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,32767,0.023071999351183575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,4095,0.017621333400408428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,65535,0.02741333345572154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,15,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1023,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,2047,0.016613333175579708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,2047,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,4095,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,4095,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,8191,0.019402666638294857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,8191,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,16383,0.024933333198229473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,16383,0.020938667158285778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,32767,0.03962666789690653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,32767,0.025066666305065155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,65535,0.05811200042565664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,65535,0.039605334401130676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,65535,0.04190933207670847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,7,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,31,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,31,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,63,0.011658667276302973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,127,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,255,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,511,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,511,0.012351999680201212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1023,0.011962667107582092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,2047,0.01540800059835116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,4095,0.017429333180189133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,4095,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,8191,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,8191,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,16383,0.03812266637881597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,16383,0.023423999547958374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,32767,0.059008002281188965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,32767,0.03759466608365377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,65535,0.057861333092053734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1,0.043920000394185386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1,0.03755199909210205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,3,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,7,0.045850664377212524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1023,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,7,0.04165333261092504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,15,0.05629866818586985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,15,0.04976533353328705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,31,0.056464001536369324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,31,0.05045866469542185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,65535,0.09945600231488545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,63,0.05041066805521647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,127,0.06645866731802623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,127,0.06046399970849355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,255,0.09728533029556274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,255,0.09248000383377075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,511,0.16054399808247885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,511,0.15260799725850424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1023,0.2874666651089986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1023,0.2754453420639038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,63,0.05596266686916351
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,2047,0.5191733439763387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,3,0.045781334241231285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,4095,1.0715680122375488
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,4095,1.0200800100962322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,3,0.014064000298579534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,3,0.01351999988158544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,7,0.013482666263977686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,7,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,15,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,15,0.01360000049074491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,31,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,31,0.013562666873137156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,63,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,63,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,127,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,127,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,255,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,511,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,511,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1023,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1023,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,2047,0.040976000328858696
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,2047,0.027285332481066387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,2047,0.5436960061391195
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,4095,0.04354133208592733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,3,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,7,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,7,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,15,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,15,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,31,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,31,0.014767999450365702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,63,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,63,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,127,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,127,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,255,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,255,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,3,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,511,0.019621333728233974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,511,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1023,0.03548266738653183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1023,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,2047,0.06148266792297363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,2047,0.04154133299986521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,4095,0.1035146713256836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,4095,0.06249066690603892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1,0.018085333208243053
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,3,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,3,0.018320000420014065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,7,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,7,0.017525333911180496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,15,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,15,0.017968000223239262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,31,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,31,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,63,0.019440000255902607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,63,0.01923199991385142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,127,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,127,0.018751999984184902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,255,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,255,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,511,0.03383466601371765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1023,0.05397866666316986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1023,0.03538133452335993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,2047,0.09575466314951579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,2047,0.05816533168156942
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,4095,0.17241066694259644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,4095,0.09521599610646565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,4095,0.062352001667022705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1,0.07081066568692525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,3,0.0870293378829956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,3,0.07281066477298737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,511,0.02364266663789749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,7,0.07668800155321757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,15,0.10624000430107117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,31,0.10585066676139832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,31,0.09541333715120952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,63,0.10769066214561462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,7,0.08800533413887024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,127,0.12590932846069336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,15,0.09500267108281453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,127,0.11583466331164043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,255,0.18731200695037842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,255,0.17512534062067667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1,0.08475733796755473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,511,0.3123573263486226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,511,0.29685866832733154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1023,0.5744746526082357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1023,0.5386026700337728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,3,0.017157333592573803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,3,0.01685333376129468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,7,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,7,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,15,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,15,0.01830400029818217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,31,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,31,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,63,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,63,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,127,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,127,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,255,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,255,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,511,0.021002667645613354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,511,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1023,0.03678400069475174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1023,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1,0.021029333273569744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,3,0.021002667645613354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,3,0.019391999890406925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,7,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,7,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,15,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,15,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,31,0.02094399929046631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,31,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,63,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,63,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,127,0.020997333029905956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,127,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1,0.01781333362062772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,255,0.018863999595244724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,511,0.035360001027584076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,511,0.02317333221435547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1023,0.054234668612480164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1023,0.035877334574858345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1,0.029264000554879505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1,0.02743999908367793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,3,0.029472000896930695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,3,0.0276053324341774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,7,0.029290666182835896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,63,0.0953653355439504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,7,0.027242665489514668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,15,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,15,0.02741866558790207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,31,0.02937600016593933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,31,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,63,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,63,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,127,0.02932800104220708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,127,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,255,0.034101332227389015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,255,0.025498665869235992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,255,0.021018666525681812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,511,0.03786666691303253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1023,0.09318932890892029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1023,0.056202664971351624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,1,0.16058666507403055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,1,0.13617066542307535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,3,0.16687466700871786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,3,0.14006933569908142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,511,0.05418133238951365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,7,0.16911999384562174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,7,0.14630400141080221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,15,0.2036479910214742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,15,0.18118399381637573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,31,0.1832480033238729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,63,0.2058453361193339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,63,0.18309332927068075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,127,0.24463999271392822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,127,0.2221013307571411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,255,0.3659306764602661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,255,0.3453226486841838
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,511,0.630026658376058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,511,0.5855253140131632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,31,0.20518932739893594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,1,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,3,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,3,0.023578666150569916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,7,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,15,0.02566933383544286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,15,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,31,0.02585600068171819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,63,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,7,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,127,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,127,0.02109866589307785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,31,0.021344001094500225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,255,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,255,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,63,0.023370665808518726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,511,0.0277813325325648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,1,0.03166399896144867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,1,0.029258665939172108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,3,0.03146666785081228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,3,0.029296000798543293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,7,0.03173866619666418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,7,0.029391999046007793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,15,0.031850665807724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,511,0.03787733366092046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,31,0.03197333216667175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,31,0.02945599953333537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,63,0.03218133250872294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,1,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,127,0.03271999955177307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,127,0.029509333272775013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,255,0.037050666908423104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,255,0.0295413335164388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,511,0.05755199988683065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,511,0.039664000272750854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,1,0.052042668064435325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,3,0.051957334081331887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,3,0.0436160018046697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,7,0.051813334226608276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,7,0.04351999859015147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,1,0.04358933369318644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,15,0.0518506666024526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,15,0.04381866753101349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,31,0.051674668987592064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,31,0.04370133578777313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,63,0.052015999952952065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,63,0.044010668992996216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,127,0.05348266661167145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,127,0.042277331153551735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,255,0.05734399954477946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,15,0.029338667790095013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,511,0.0960053304831187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,63,0.02959999938805898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,3,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,255,0.045567999283472695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,15,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,511,0.062394668658574425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1,0.010570666442314783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,63,0.015461333096027374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,7,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,63,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,127,0.015333333363135656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,255,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,255,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,511,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,511,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,31,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1023,0.031397332747777305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1023,0.029781334102153778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,2047,0.0518506666024526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,31,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,4095,0.09113599856694539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,4095,0.08718933661778767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,8191,0.17112533251444498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,8191,0.1628106633822123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,16383,0.32843732833862305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,16383,0.31411200761795044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,2047,0.04786666731039683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,32767,0.7038453420003256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,65535,1.6844107309977214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,32767,0.6444373528162638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,7,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1,0.01202133297920227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,127,0.01191466674208641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,511,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,511,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,2047,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,65535,1.4549652735392253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,255,0.01179733375708262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,8191,0.021589333812395733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,8191,0.019600000232458115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,16383,0.025349333882331848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,16383,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,4095,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,4095,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,32767,0.027514666318893433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,65535,0.06043200194835663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,2047,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,3,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,32767,0.041759997606277466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,15,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,31,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,65535,0.04367466767628988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,511,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1023,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,2047,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,4095,0.01798933371901512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,4095,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,8191,0.02162666618824005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,8191,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,16383,0.03984000037113825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,16383,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,32767,0.05922666688760122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,32767,0.03772799919048945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,65535,0.09930666287740071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,15,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,15,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,63,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,511,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1023,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,65535,0.05763733386993408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1023,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,2047,0.017535999417304993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,2047,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,4095,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,4095,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,8191,0.03961600114901861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,8191,0.023557332654794056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,16383,0.05890666445096334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,16383,0.037647999823093414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,32767,0.10126399993896484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,32767,0.0581226646900177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,65535,0.1832373340924581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,65535,0.09752532839775085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,1,0.26099733511606854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,3,0.3246293266614278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,3,0.2715253432591756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,7,0.2841386596361796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,1,0.31243733565012616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,15,0.4040106534957886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,15,0.3569493293762207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,31,0.4005013306935628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,31,0.35769601662953693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,63,0.4043999910354614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,63,0.3590933481852214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,7,0.3327573339144389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,127,0.48013333479563397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,127,0.43466134866078693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,255,0.7510933081309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,1,0.033557333052158356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,3,0.0418453315893809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,3,0.03380800038576126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,7,0.03977066775163015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,7,0.033514666060606636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,1,0.040394666294256844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,15,0.039749334255854286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,15,0.03492266684770584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,31,0.03962666789690653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,31,0.03440533330043157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,63,0.039477333426475525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,127,0.039461334546407066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,127,0.035375999907652535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,255,0.6778506437937418
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,255,0.04140799989302953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,255,0.03538133452335993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,1,0.05452266832192739
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,1,0.0480320006608963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,3,0.054101333022117615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,3,0.047914668917655945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,7,0.05422399938106537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,7,0.04783466458320618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,15,0.05444266895453135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,15,0.04791999856630961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,31,0.05398400127887726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,31,0.04785066843032837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,63,0.05392533540725708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,63,0.04781866570313772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,63,0.035962666074434914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,127,0.05622399846712748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,127,0.047914668917655945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,255,0.06158400078614553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,1,0.0930613378683726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,255,0.050767997900644936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,1,0.07876266539096832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,3,0.09134933352470398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,3,0.07955199976762135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,7,0.09223467111587524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,7,0.07870933413505554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,15,0.09146666526794434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,15,0.07876266539096832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,31,0.09277333815892537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,31,0.07815466821193695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,63,0.091648002465566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,63,0.07900266846021016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,127,0.09298666318257649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,127,0.07815999786059062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,255,0.10085866848627727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,1,0.6194026470184326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,1,0.5129813353220621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,3,0.6442826588948568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,255,0.08096000055472057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,3,0.5375039974848429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,7,0.6570613384246826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,7,0.562117338180542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,15,0.7966612974802653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,15,0.698965311050415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,31,0.7960533301035563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,31,0.7073706785837809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,63,0.8055040041605631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,63,0.6993652979532877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,127,0.9743146896362305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,127,0.8653066953023275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,1,0.06849599877993266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,1,0.05952533086140951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,3,0.06733866532643636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,3,0.05825600028038025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,7,0.06819200019041698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,7,0.05797866483529409
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,15,0.06687999765078227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,15,0.05905066430568695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,31,0.06853866577148438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,31,0.058677335580190025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,63,0.06702400247255962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,63,0.05832533538341522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,127,0.06878933310508728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,127,0.05807999769846598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,1,0.09808533390363057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,1,0.08621866504351298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,3,0.09742933511734009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,3,0.0851146678129832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,7,0.09788800279299419
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,7,0.08555733164151509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,15,0.08513599634170532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,15,0.09777599573135376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,31,0.097653329372406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,31,0.08638933300971985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,63,0.09775466720263164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,63,0.08571199576059978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,1,0.1696853240331014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,3,0.1709173321723938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,1,0.14444266756375632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,3,0.14453867077827454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,7,0.17010132471720377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,7,0.14447999993960062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,127,0.09966933727264404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,127,0.08733333150545756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,15,0.17065600554148355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,15,0.14447999993960062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,31,0.17076265811920166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,31,0.14458666245142618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,63,0.17085866133371988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1,0.015354666858911514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,63,0.14493866761525473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,3,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,127,0.17122666041056314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,7,0.01351999988158544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,7,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,127,0.14267200231552124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,31,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,31,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,63,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,63,0.019530666371186573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,127,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,127,0.019098666807015736
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,255,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,3,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,511,0.03179733455181122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,15,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1023,0.05013866722583771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1023,0.05147733290990194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,2047,0.09134933352470398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,2047,0.08872532844543457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,4095,0.1710346738497416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,4095,0.16274666786193848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,15,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,511,0.033514666060606636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,8191,0.3100000023841858
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,16383,0.6660213470458984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,16383,0.602346658706665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,255,0.023370665808518726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,8191,0.3322239915529887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,32767,1.519034703572591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,15,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,31,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,32767,2.36080535252889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,511,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1023,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,2047,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,4095,0.017674667139848072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,4095,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,8191,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,8191,0.02091199904680252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,16383,0.04009599983692169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,2047,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,32767,0.06069866816202799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,32767,0.04359466830889384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,15,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,16383,0.026613332331180573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,63,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,255,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,511,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1023,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1023,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,2047,0.0184906671444575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,2047,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,4095,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,4095,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,8191,0.0397173340121905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,16383,0.05992533266544342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,16383,0.03976533313592275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,32767,0.10040000081062317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,8191,0.024720000723997753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,3,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,3,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,7,0.01181866725285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,15,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,15,0.012335999558369318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,31,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,511,0.012165332833925882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1023,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,32767,0.05864533285299937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,2047,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,4095,0.039333333571751915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,8191,0.05967999994754791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,8191,0.03839999934037527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,16383,0.09991466999053955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1023,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,2047,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,4095,0.023082666099071503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,32767,0.18246400356292725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,32767,0.0974026620388031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,3,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,7,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,16383,0.05831466615200043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,15,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,15,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,31,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,63,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,127,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,511,0.012234666695197424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,511,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,511,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1023,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,2047,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,2047,0.015861333658297855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,2047,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,4095,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,4095,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,8191,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,8191,0.019776000330845516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,2047,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,4095,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,16383,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,16383,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,8191,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,8191,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,16383,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,32767,0.014175999909639359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,32767,0.04375466704368591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,32767,0.035717333356539406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,3,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,32767,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,16383,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,255,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,511,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,511,0.011648000528415045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1023,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,2047,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,2047,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,2047,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,2047,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,4095,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,4095,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,4095,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,511,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,127,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,8191,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,8191,0.022426667312781017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,16383,0.013557333499193192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,16383,0.039605334401130676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,16383,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,16383,0.0312266672650973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,8191,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,8191,0.024319998919963837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,32767,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,32767,0.06010133524735769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,32767,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,32767,0.05198400219281515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,255,0.010490667074918747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,511,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,511,0.012549333274364471
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1023,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1023,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1023,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,2047,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,2047,0.01821333294113477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,2047,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,4095,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,4095,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,4095,0.01451733335852623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,8191,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,8191,0.038805333276589714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,8191,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,16383,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,16383,0.059194669127464294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,16383,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,16383,0.05222400029500326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,4095,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,32767,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,32767,0.09920000036557515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,32767,0.013898666948080063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1,0.01258133351802826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,32767,0.08731733759244283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,3,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,3,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,8191,0.03149333347876867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,7,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,15,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,15,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,3,0.012666666259368261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,15,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,31,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,31,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,31,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,127,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,127,0.012533333152532578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,127,0.013658666362365087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,255,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,255,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,255,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,255,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,511,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,31,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1023,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1023,0.016362667083740234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1023,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,2047,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,2047,0.023061332603295643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,4095,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,511,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,4095,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,4095,0.031152000029881794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,15,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,8191,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,8191,0.06071466704209646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,8191,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,8191,0.052671998739242554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,16383,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,16383,0.10175466537475586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,16383,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,16383,0.08719467123349507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,32767,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,32767,0.1852160096168518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,4095,0.03978666663169861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,32767,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,32767,0.15804800391197205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,2047,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,7,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,63,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,511,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1023,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1023,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,2047,0.011765333513418833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,2047,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,2047,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,2047,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,4095,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,4095,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,4095,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,4095,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,8191,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,8191,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,8191,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,8191,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,16383,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,255,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,16383,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,16383,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,32767,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,32767,0.02718399961789449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,32767,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,32767,0.027269333600997925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,65535,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,16383,0.018922666708628338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,63,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,65535,0.011744000017642975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,65535,0.029306667546431225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,131071,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,65535,0.030453334252039593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,131071,0.033674667278925575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,3,0.010469333579142889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,7,0.012293333808581034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,131071,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,15,0.010341333225369453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,31,0.010378666842977205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,31,0.010522666076819101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,63,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,63,0.010490667074918747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,127,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,255,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,511,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,511,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,511,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1023,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1023,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,2047,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,2047,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,2047,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,2047,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,4095,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,4095,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,4095,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,4095,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,8191,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,8191,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,131071,0.03331200033426285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,8191,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,16383,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,16383,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1023,0.011813333878914515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,16383,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,32767,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,8191,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,32767,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,32767,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,65535,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,65535,0.023237332701683044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,32767,0.02161066730817159
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,65535,0.023232000569502514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,16383,0.01982933282852173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,131071,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,131071,0.025744001070658367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,65535,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,131071,0.027162666122118633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,3,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,31,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,63,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,63,0.009786666681369146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,63,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,63,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,127,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,255,0.009930666536092758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,511,0.012175999581813812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1023,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1023,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,2047,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,2047,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,2047,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,2047,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,4095,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,4095,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,4095,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,4095,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,7,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,8191,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,8191,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,8191,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,16383,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,16383,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,16383,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,16383,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,32767,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,32767,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,32767,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,32767,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,131071,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,65535,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,65535,0.021205333371957142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,65535,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,65535,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,8191,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,131071,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,131071,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,131071,0.027242665489514668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,3,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,3,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,7,0.010597333312034607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,15,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,15,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,31,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,127,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,131071,0.02743999908367793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,255,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,63,0.011567999919255575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,511,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,511,0.01166933278242747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1023,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1023,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1023,0.01219733307758967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,2047,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,2047,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,7,0.01044800008336703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,2047,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,4095,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,4095,0.013557333499193192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,4095,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,8191,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,8191,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,8191,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,8191,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,16383,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,16383,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,16383,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,2047,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,32767,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,4095,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,32767,0.019648000597953796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,32767,0.018837332725524902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,65535,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,16383,0.017445333302021027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,65535,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,65535,0.023183998962243397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,32767,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,65535,0.02312533309062322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,131071,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,131071,0.0394400010506312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,3,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,7,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,131071,0.03166933357715607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,15,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,15,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,31,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,127,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,131071,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1023,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,7,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1023,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1023,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,2047,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,2047,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,2047,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,4095,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,4095,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,4095,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,4095,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,8191,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,8191,0.01682666689157486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,8191,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,8191,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,16383,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,16383,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,16383,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,16383,0.019925333559513092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,32767,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,511,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,32767,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,32767,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,65535,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,65535,0.02422400067249934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,32767,0.023168000082174938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,65535,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,65535,0.025445332129796345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,131071,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,131071,0.029317334294319153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,131071,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,131071,0.028431999186674755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,3,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,3,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,15,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,15,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,127,0.010464000205198923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,127,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,255,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,511,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,511,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,511,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1023,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1023,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,2047,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,2047,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,2047,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,2047,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,4095,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,4095,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,4095,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,4095,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,8191,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,8191,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,8191,0.016672000288963318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,16383,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,16383,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,16383,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,16383,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,32767,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,32767,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,32767,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,32767,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,8191,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,65535,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,65535,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,65535,0.021295999487241108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,131071,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,131071,0.0273333340883255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,131071,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,131071,0.027514666318893433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,65535,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,7,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,31,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,63,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,127,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,511,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,511,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1023,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,255,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,255,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,2047,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,2047,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,2047,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,2047,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,4095,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1023,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,4095,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,4095,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,8191,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,8191,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,8191,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,8191,0.015482666591803232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,16383,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,16383,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,16383,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,16383,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,4095,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,32767,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,32767,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1023,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,32767,0.018922666708628338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,65535,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,65535,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,65535,0.023317334552605946
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,32767,0.012586666891972223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,131071,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,65535,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,131071,0.03154666721820831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,3,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,131071,0.039893334110577904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,15,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,131071,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,63,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,63,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,127,0.010346666599313417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,511,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,511,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1023,0.010527999450763067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1023,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,2047,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,2047,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,2047,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,4095,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,4095,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,4095,0.012319999436537424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,4095,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,8191,0.01201066623131434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,8191,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,8191,0.01543466622630755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,16383,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,16383,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,16383,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,16383,0.0179626668492953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,32767,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,32767,0.023157333334287006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,8191,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,32767,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,32767,0.022005334496498108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,65535,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,65535,0.03597866743803024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,65535,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,65535,0.029498666524887085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,2047,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,131071,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,131071,0.05801600217819214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,15,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,63,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,511,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,131071,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,511,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1023,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,2047,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,2047,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,2047,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,2047,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1023,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,4095,0.0210506667693456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,131071,0.052058666944503784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,4095,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,4095,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,8191,0.014650666465361914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,8191,0.026389333109060924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,8191,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,8191,0.025221332907676697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,16383,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,16383,0.043525333205858864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,16383,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,3,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,3,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,4095,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1,0.012229333321253458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,63,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,63,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,511,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,511,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1023,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,63,0.0122079998254776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1023,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,16383,0.035536001125971474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,2047,0.01800000046690305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,2047,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,2047,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,4095,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,4095,0.023226665953795116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,4095,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,4095,0.02142400046189626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,8191,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,8191,0.03873066604137421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,2047,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,8191,0.03138133386770884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,16383,0.014757333944241205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,16383,0.05961599946022034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,16383,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,16383,0.051962668697039284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,3,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,3,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,3,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,3,0.011472000430027643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,7,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,7,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,7,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,15,0.012698666503032049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,15,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,15,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,15,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,31,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,31,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,31,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,8191,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,63,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,63,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,63,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,127,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,127,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,255,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,31,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,255,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,63,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,511,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,511,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,511,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1023,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1023,0.015520000209410986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,2047,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,2047,0.023775999744733173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,255,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,2047,0.023039999107519787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,4095,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,4095,0.03990400085846583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,4095,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,4095,0.03149333347876867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,8191,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,8191,0.06121600170930227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1023,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,8191,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,16383,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,16383,0.10288533568382263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,16383,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,16383,0.08731733759244283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1,0.013823999712864557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,3,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,3,0.014357333381970724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,3,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,3,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,7,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,7,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,7,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,7,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,15,0.01360000049074491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,15,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,15,0.01357866699496905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,15,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,31,0.01404800017674764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,31,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,31,0.01351999988158544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,31,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,63,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,63,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,63,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,127,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,127,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,127,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,255,0.014287999520699183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,255,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,511,0.01414399966597557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,511,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,511,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,511,0.01716800034046173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1023,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1023,0.021957332889238994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1023,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1023,0.020986666282018025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,2047,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,2047,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,2047,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,2047,0.03148799886306127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,4095,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,4095,0.0614879975716273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,4095,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,4095,0.05392533540725708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,8191,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,8191,0.10373333096504211
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,8191,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,8191,0.08893866340319316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,16383,0.016917333006858826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,16383,0.18745599190394083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,16383,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,63,0.015856000284353893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1,0.011770666887362799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1,0.011690666278203329
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,3,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,3,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,3,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,7,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,7,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,15,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,31,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,63,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,127,0.011519999553759893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,511,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1023,0.010527999450763067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,2047,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,2047,0.013381333400805792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,2047,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,8191,0.05249066650867462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,4095,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,4095,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,4095,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,4095,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,8191,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,8191,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,2047,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,8191,0.017082666357358296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,16383,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,16383,0.0200853335360686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,16383,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,16383,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,32767,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,32767,0.021477334201335907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,32767,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,32767,0.021898667017618816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,65535,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,65535,0.023567999402681988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,65535,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,65535,0.02332266668478648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,16383,0.15868799885114035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,131071,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,131071,0.03136533250411352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,131071,0.01462399959564209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,3,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,131071,0.02972800036271413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,7,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,7,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,63,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,255,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,511,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,511,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1023,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1023,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1023,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1023,0.01227733368674914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,63,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,2047,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,4095,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,4095,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,4095,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,4095,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,8191,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,8191,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,8191,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,8191,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,16383,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,2047,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,16383,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,16383,0.018735999862353008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,32767,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,32767,0.021055998901526134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,32767,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,16383,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,65535,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,65535,0.025653332471847534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,65535,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,65535,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,32767,0.02027200038234393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,131071,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,131071,0.04115733255942663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,131071,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,131071,0.03332799921433131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,15,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,31,0.010543999572594961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,31,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,63,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,127,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,127,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,255,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,255,0.011519999553759893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,511,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,511,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,511,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,511,0.01163200040658315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1023,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1023,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,2047,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,2047,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,2047,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,4095,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,4095,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,4095,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,8191,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,8191,0.01871466636657715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,8191,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,8191,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,16383,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,16383,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,16383,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,16383,0.020970667401949566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,32767,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,32767,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,32767,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,32767,0.023306667804718018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,65535,0.03974399964014689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,65535,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,65535,0.03165333221356074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,65535,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,2047,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,131071,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,131071,0.06018133461475372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1,0.010543999572594961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,131071,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,131071,0.05217599868774414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,3,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,7,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,7,0.010527999450763067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,31,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,63,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,127,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,255,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,255,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,511,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1023,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1023,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1023,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,2047,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,2047,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,4095,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,4095,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,4095,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,4095,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,8191,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,8191,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,2047,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,8191,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,8191,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,16383,0.013440000514189402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,16383,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,16383,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,32767,0.012666666259368261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,32767,0.037791999677817024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,32767,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,32767,0.029530666768550873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,16383,0.023306667804718018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,65535,0.05826666454474131
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,65535,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,65535,0.052485331892967224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,65535,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,131071,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,131071,0.10366933544476827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1,0.011589333415031433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,131071,0.08880533774693807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,131071,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,15,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,63,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,127,0.011706666400035223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,255,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,255,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,255,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,255,0.011813333878914515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,511,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,511,0.012666666259368261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1023,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1023,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1023,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,511,0.0124746672809124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,2047,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,2047,0.01882133384545644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,4095,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,4095,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,4095,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,4095,0.023589332898457844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1023,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,8191,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,2047,0.019493332753578823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,8191,0.014229333649079004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1,0.012186666329701742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,8191,0.035402665535608925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1,0.011594666788975397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,3,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,3,0.011941333611806234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,3,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,7,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,7,0.01191466674208641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,8191,0.042250668009122215
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,15,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,15,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1,0.01249066616098086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,15,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,31,0.012362666428089142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,31,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,31,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,31,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,63,0.012181332955757776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,63,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,63,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,15,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,127,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,127,0.012351999680201212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,255,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,255,0.012560000022252401
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,255,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,255,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,511,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,511,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,511,0.013818666338920593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,127,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1023,0.016538667182127636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,2047,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,2047,0.01764800027012825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,2047,0.025653332471847534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,2047,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,4095,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,4095,0.04090133309364319
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,4095,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,4095,0.031514666974544525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1023,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,8191,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,8191,0.061749334136645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,8191,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,3,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,3,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,3,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,3,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,7,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,7,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,7,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,7,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,15,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,15,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,15,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,15,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,31,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,31,0.014767999450365702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,31,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,63,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,63,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,63,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,127,0.01358933374285698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,127,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,127,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,255,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,255,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,8191,0.054144000013669334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,255,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,511,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,511,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,511,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,511,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1023,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1023,0.023039999107519787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1023,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1023,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,2047,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,2047,0.03944533318281174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,255,0.014783999572197596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,2047,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,2047,0.03278933217128118
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,4095,0.06170133252938589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,4095,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,4095,0.053871999184290566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,8191,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,8191,0.10459733009338379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,8191,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,8191,0.08914132912953694
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,3,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,3,0.01811733345190684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,3,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,7,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,7,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,7,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,15,0.01783466711640358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,15,0.017887999614079792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,15,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,15,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,31,0.017829333742459614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,3,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,7,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,31,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,63,0.017632000148296356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,63,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,63,0.01775466650724411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,63,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,127,0.018031999468803406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,127,0.017680000513792038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,31,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,255,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,255,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,255,0.017509333789348602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,255,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,127,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,31,0.01825599993268649
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,127,0.01782400036851565
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,511,0.0215786670645078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1023,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1023,0.034330666065216064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,511,0.01757866640885671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,511,0.023056000471115112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1023,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,511,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,2047,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,2047,0.05726400017738342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,2047,0.020346666375796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,2047,0.05020266771316528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,4095,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,4095,0.09494933485984802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,4095,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,4095,0.08211733400821686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1023,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,8191,0.02103466788927714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,8191,0.16800000270207724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,8191,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,8191,0.14243732889493307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1,0.013445333888133367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,3,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,3,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,3,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,3,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,7,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,7,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,7,0.012618667135636011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,15,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,15,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,15,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,15,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,31,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,31,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,31,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,31,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,63,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,7,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,63,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,63,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,127,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,127,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,127,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,255,0.013493333011865616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,255,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,255,0.01340266689658165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,255,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,511,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,511,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,511,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,511,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1023,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1023,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1023,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1023,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,2047,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,2047,0.025818665822347004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,2047,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1,0.01357866699496905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,3,0.013669333110253016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,3,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,3,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,3,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,7,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,7,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,7,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,7,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,15,0.014682666709025701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,2047,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,15,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,15,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,15,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,31,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,31,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,31,0.013616000612576803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,31,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,63,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,63,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,63,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,63,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,127,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,127,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,127,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,127,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,255,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,255,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,255,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,255,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,511,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,511,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,511,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,511,0.01754133279124896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1023,0.015290666371583939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1023,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1023,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1023,0.01562133307258288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,63,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,2047,0.01747200017174085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,2047,0.03955200066169103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,2047,0.03178133318821589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,3,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,3,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,3,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,3,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,7,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,7,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,7,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,7,0.017429333180189133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,15,0.017605333278576534
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,15,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,15,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,15,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,31,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,31,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,31,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,31,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,63,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,63,0.018320000420014065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,63,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,63,0.017535999417304993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,127,0.01842133328318596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,127,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,127,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,127,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,255,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,255,0.017637333522240322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,255,0.01756799966096878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,255,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,511,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,511,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,511,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,511,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1023,0.01841066653529803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1023,0.0353973334034284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1023,0.017535999417304993
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1023,0.02932800104220708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,2047,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,2047,0.05825600028038025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,2047,0.020309332758188248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,2047,0.05128533144791921
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1,0.02311466634273529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1,0.025072000920772552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,3,0.025301332275072735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,3,0.025370667378107708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,3,0.023258666197458904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,3,0.025114665428797405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,7,0.024986666937669117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,7,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,7,0.023061332603295643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,7,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,15,0.025642665723959606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,15,0.02402133246262868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,15,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,31,0.025216000775496166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,31,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,31,0.023621333142121632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,31,0.023258666197458904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,63,0.025648000339667004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,63,0.02515200028816859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,63,0.025040000677108765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,63,0.02317333221435547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,15,0.025055999557177227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,127,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,127,0.023530667026837666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,2047,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,127,0.02346666653951009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,255,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,255,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,255,0.023391999304294586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,255,0.023285334308942158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,511,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,511,0.0378506655494372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,127,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,511,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,511,0.03139200061559677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1023,0.027477333943049114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1023,0.05594133337338766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1023,0.02366400013367335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1023,0.05042133231957754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,2047,0.02932800104220708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,2047,0.09735999504725139
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,2047,0.027461332579453785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,2047,0.08468266328175862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1,0.015530666957298914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,3,0.015360000232855478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,3,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,3,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,3,0.01588800052801768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,7,0.01534933348496755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,15,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,15,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,15,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,15,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,31,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,7,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,31,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,31,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,7,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,63,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,63,0.015344000111023584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,63,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,127,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,127,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,127,0.015397333850463232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,127,0.014837333311637243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,255,0.015493333339691162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,255,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,255,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,511,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,511,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,511,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,511,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1023,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1023,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1023,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1,0.018207999567190807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1,0.018672000616788864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,31,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,3,0.018453333526849747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,3,0.018373332917690277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,3,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,3,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,7,0.01923199991385142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,7,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,7,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,7,0.018373332917690277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1023,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,15,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,15,0.018885333091020584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,15,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,31,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,31,0.01877333347996076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,31,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,31,0.017525333911180496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,63,0.018672000616788864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,63,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,63,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,63,0.01802666609485944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,127,0.01887999971707662
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,15,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,127,0.018757333358128864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,127,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,255,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,255,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,255,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,511,0.01884799947341283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,511,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,511,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,511,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,127,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1023,0.0354720006386439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1023,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1023,0.029663999875386555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1,0.025301332275072735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1,0.025568000972270966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1,0.02437866727511088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,3,0.02569599946339925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,3,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1023,0.01894933357834816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,3,0.02500266581773758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,3,0.023578666150569916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,7,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,7,0.023621333142121632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,7,0.023402666052182514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,15,0.025642665723959606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,15,0.025445332129796345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,15,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,15,0.025120000044504803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,31,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,31,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,31,0.02510400116443634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,31,0.02536533276240031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,63,0.025519999365011852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,63,0.024469333390394848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,63,0.02348266790310542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,127,0.025392000873883564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,127,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,127,0.023210667073726654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,127,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,255,0.02510933329661687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,255,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,255,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,255,0.025050667424996693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,511,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,511,0.037674665451049805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,7,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,511,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,511,0.03332266708215078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1023,0.027221334477265675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1023,0.05630933245023092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,63,0.026026666164398193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1023,0.05008000135421753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1,0.040778666734695435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1,0.0379573330283165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1,0.03771200031042099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,3,0.041482667128245033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,3,0.0405173326532046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1023,0.025583999852339428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,3,0.03763733307520548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1,0.03976533313592275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,7,0.04051200052102407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,7,0.037402667105197906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,7,0.03745066622893015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,15,0.041333332657814026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,15,0.03965866565704346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,15,0.03754133234421412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,15,0.03755733370780945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,7,0.041493333876132965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,31,0.039642666776975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,31,0.03748800108830134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,31,0.03748266647259394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,63,0.04188799858093262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,63,0.04053866614898046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,63,0.03761066744724909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,63,0.0374293327331543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,127,0.04002666721741358
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,127,0.039621333281199135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,127,0.03753600021203359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,127,0.03735466549793879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,255,0.04203199843565623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,3,0.0376800000667572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,255,0.03762666632731756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,31,0.041008000572522484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,511,0.04377600053946177
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,511,0.06214933097362518
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,511,0.03787733366092046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,511,0.054560000697771706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,255,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1023,0.04179200033346812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1023,0.0969546635945638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,255,0.03759466608365377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,3,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,63,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1023,0.08498133222262065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,511,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,511,0.012229333321253458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1023,0.011754666765530905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1023,0.037615999579429626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,2047,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,2047,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,2047,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,4095,0.013679999858140945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,4095,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,4095,0.01710933322707812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,8191,0.01877333347996076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,8191,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,8191,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1023,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,16383,0.019487999379634857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,16383,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,16383,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,4095,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,32767,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,32767,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,32767,0.021290667355060577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,16383,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,65535,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,65535,0.027647999425729115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,32767,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,65535,0.025087999800841015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,131071,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,131071,0.0444213350613912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,65535,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,131071,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,7,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,15,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,15,0.010512000570694605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,131071,0.03751466671625773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,31,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,63,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,63,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,63,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,511,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1023,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,2047,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,2047,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,2047,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,2047,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,4095,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,4095,0.015930666277805965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,4095,0.013381333400805792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,8191,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,8191,0.018778666853904724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,8191,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,8191,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,16383,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,16383,0.020975999534130096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,16383,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,16383,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,32767,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,32767,0.02513066679239273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,32767,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,32767,0.023578666150569916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,4095,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,65535,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,65535,0.04001066585381826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,65535,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,65535,0.0314026673634847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,131071,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,131071,0.06014933188756307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,131071,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,3,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,3,0.012080000092585882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,131071,0.053130666414896645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,7,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,15,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,255,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,255,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,511,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1023,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1023,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1023,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,2047,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,2047,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,2047,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,4095,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,4095,0.015594666202863058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,4095,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,4095,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,8191,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,8191,0.017978666971127193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,8191,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,8191,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,3,0.011792000383138657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,16383,0.023071999351183575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,16383,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,16383,0.021322667598724365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,32767,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,32767,0.03782933453718821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,32767,0.02938666691382726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,16383,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,65535,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,65535,0.05996799965699514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,65535,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,65535,0.051856001218159996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,131071,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,131071,0.1035146713256836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,131071,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,32767,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,131071,0.08880533774693807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,7,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,31,0.01201066623131434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,255,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,255,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,511,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,511,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1023,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1023,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1023,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,2047,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,2047,0.016730666160583496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,2047,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,2047,0.01611199975013733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,4095,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,4095,0.019567999988794327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,4095,0.01810666670401891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,8191,0.013424000392357508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,8191,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,8191,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,16383,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,16383,0.03791466603676478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,16383,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,16383,0.031173333525657654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,4095,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,32767,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,32767,0.060122668743133545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,32767,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,8191,0.02364266663789749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,32767,0.05167999863624573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,65535,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,65535,0.10132267077763875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,65535,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,65535,0.08681600292523702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,131071,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,131071,0.015583999454975128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,131071,0.15807466705640158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,1,0.019589333484570186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,1,0.020970667401949566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,1,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,1,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,3,0.021136000752449036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,3,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,3,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,131071,0.18755199511845908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,3,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,7,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,7,0.019482667247454327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,7,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,15,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,15,0.021029333273569744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,15,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,15,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,31,0.021242665747801464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,31,0.019354666272799175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,31,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,31,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,7,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,63,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,63,0.019189332922299702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,63,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,63,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,127,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,127,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,127,0.019466667125622433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,127,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,255,0.021568000316619873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,255,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,255,0.02109866589307785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,255,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,511,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,511,0.02626666675011317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,511,0.020928000410397846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,511,0.02390933285156886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,1,0.02714666724205017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,1,0.027434666951497395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,1,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,1,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,3,0.027141332626342773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,3,0.02720000098148982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,3,0.025616000096003216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,3,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,7,0.027285332481066387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,7,0.027034667630990345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,7,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,7,0.025775998830795288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,15,0.027119999130566914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,15,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,15,0.025781333446502686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,15,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,31,0.025237334271272022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,31,0.027493332823117573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,31,0.025253333151340485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,31,0.025221332907676697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,63,0.027248000105222065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,63,0.02569066733121872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,63,0.025349333882331848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,63,0.02569066733121872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,127,0.025445332129796345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,127,0.027098665634791057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,127,0.02569066733121872
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,127,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,255,0.027306665976842243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,255,0.02720000098148982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,255,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,255,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,511,0.03789333254098892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,511,0.027658666173617046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,511,0.03341866781314214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,1,0.041509332756201424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,1,0.041589332123597465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,1,0.037818667789300285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,1,0.03781333317359289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,3,0.04159999887148539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,3,0.04162666698296865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,511,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,3,0.03762666632731756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,3,0.03772266705830892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,7,0.04173333446184794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,7,0.04182399809360504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,7,0.03769599894682566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,15,0.04185600082079569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,15,0.04159999887148539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,15,0.03800000001986822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,15,0.03764266769091288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,31,0.0418453315893809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,31,0.04159999887148539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,31,0.0402399996916453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,7,0.03856533269087473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,31,0.037834666669368744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,63,0.04158399999141693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,63,0.04161066561937332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,63,0.03826133410135905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,63,0.037717332442601524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,127,0.04038399954636892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,127,0.03977599988381068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,127,0.038032000263532005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,127,0.03740799923737844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,255,0.043925335009892784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,255,0.04293866455554962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,255,0.04005866746107737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,255,0.037477334340413414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,511,0.043663998444875084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,511,0.062165334820747375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,511,0.04005333284536997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,511,0.05428266525268555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,1,0.07237866520881653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,1,0.07055466870466869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,1,0.06496533254782359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,3,0.07235200206438701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,3,0.07055466870466869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,3,0.06531733274459839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,3,0.06474133332570393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,7,0.07225066423416138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,7,0.07256533205509186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,7,0.06473066906134288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,1,0.06413866579532623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,15,0.07057066758473714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,15,0.07216533521811168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,15,0.0642986645301183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,15,0.06461866696675618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,31,0.07221333185831706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,31,0.07100800176461537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,31,0.06424533327420552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,7,0.06448533137639363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,63,0.07162666817506154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,63,0.07106133302052815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,63,0.06449600060780843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,63,0.06454933186372121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,127,0.07229866584142049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,127,0.07271466652552287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,31,0.064410666624705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,255,0.07644799848397572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,255,0.07257066667079926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,255,0.06658666829268138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,255,0.06637333333492279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,127,0.06405866642793019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,511,0.07625066737333934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,511,0.1114026705423991
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,511,0.06670400003592174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,511,0.09714667002360027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,1,0.031311998764673867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,127,0.06445333361625671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,1,0.03196800003449122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,1,0.029359998802344005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,1,0.029343999922275543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,3,0.03120533376932144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,3,0.03145600110292435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,3,0.029322666426499683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,3,0.02972800036271413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,7,0.031498665610949196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,7,0.03141333411137263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,7,0.029717333614826202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,15,0.03163733333349228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,15,0.02972800036271413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,7,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,31,0.03134933362404505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,31,0.031632001201311745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,31,0.029696000119050343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,31,0.02958933264017105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,63,0.031370667119820915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,63,0.031317333380381264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,15,0.03192000091075897
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,15,0.029989334444204967
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,63,0.029946667452653248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,127,0.03147733211517334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,127,0.02940266579389572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,127,0.03158933420976003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,127,0.029951999584833782
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,255,0.03369066615899404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,255,0.03146666785081228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,255,0.03134933362404505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,1,0.043840001026789345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,255,0.02941333254178365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,63,0.029877332349618275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,1,0.04409066836039225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,1,0.03962666789690653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,1,0.04166933397452036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,3,0.043968002001444496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,3,0.04397333165009817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,3,0.039919999738534294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,3,0.041802664597829185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,7,0.04367466767628988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,7,0.043893332282702126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,7,0.0395359992980957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,7,0.04121066629886627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,15,0.043562665581703186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,15,0.03990933299064636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,15,0.041850666205088295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,31,0.04362666606903076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,31,0.04399466514587402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,31,0.0417546679576238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,31,0.03979733337958654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,63,0.04366933306058248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,63,0.04390933116277059
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,63,0.039887999494870506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,15,0.04376000165939331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,63,0.04161600023508072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,127,0.043578664461771645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,127,0.04191466669241587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,127,0.041519999504089355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,127,0.041589332123597465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,255,0.04601066807905833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,255,0.04568000137805939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,255,0.041797334949175514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,255,0.04168533285458883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,1,0.07468266785144806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,1,0.07252266506354015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,1,0.06797866523265839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,1,0.06811200082302094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,3,0.07483733197053273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,3,0.07234133283297221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,3,0.06830400228500366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,3,0.0683840016523997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,7,0.07449066638946533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,7,0.07400533556938171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,7,0.06844266752401988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,7,0.0682239979505539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,15,0.07462933162848155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,15,0.07431999842325847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,15,0.06846400101979573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,15,0.06843199829260509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,31,0.07443733513355255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,31,0.0739573339621226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,31,0.06836799780527751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,31,0.06811200082302094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,63,0.07432533303896587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,63,0.07449066638946533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,63,0.06840000053246816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,63,0.0671253353357315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,127,0.07478933533032735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,127,0.07496533294518788
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,127,0.06654933094978333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,255,0.07681066791216533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,255,0.06919466455777486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,255,0.07046400010585785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,1,0.13531733552614847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,1,0.12128000458081563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,127,0.06713599960009257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,255,0.07810666660467784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,3,0.13501866658528647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,3,0.1357919971148173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,3,0.12112533052762349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,3,0.12191466490427653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,1,0.13539733489354452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,7,0.13404267032941183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,7,0.13555733362833658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,7,0.12018666664759318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,7,0.12165333827336629
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,15,0.13430399696032205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,1,0.12198399504025777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,15,0.1362773378690084
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,15,0.12011200189590454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,15,0.12190933028856914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,31,0.13598400354385376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,31,0.1344106694062551
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,31,0.12149866422017415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,63,0.13590932885805765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,63,0.1345866620540619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,31,0.12196266651153564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,63,0.12191466490427653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,63,0.12185600399971008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,127,0.13359999656677246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,127,0.1329813301563263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,127,0.12363200386365254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,127,0.12369599938392639
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,255,0.1400159994761149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,255,0.1344319979349772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,255,0.1267306705315908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,255,0.1241439978281657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,3,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,3,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,7,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,7,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1023,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1023,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,2047,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,2047,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,2047,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,2047,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,4095,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,4095,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,4095,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,8191,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,8191,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,8191,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,8191,0.01923199991385142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,16383,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,16383,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,16383,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,32767,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1023,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,32767,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,32767,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,16383,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,65535,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,65535,0.04322133461634318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,32767,0.027301333844661713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,65535,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,65535,0.033914667864640556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,3,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,3,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,3,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,7,0.01051733394463857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,7,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,3,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,31,0.012565333396196365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,63,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,127,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,127,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,255,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,255,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,255,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,511,0.010389333590865135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,511,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,511,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,15,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1023,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,2047,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,2047,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,2047,0.015504000087579092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,4095,0.015754666179418564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,4095,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,4095,0.015728000551462173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,8191,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,8191,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,8191,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,8191,0.018351999421914417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,16383,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,2047,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,16383,0.02365333338578542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,16383,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,16383,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,32767,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,32767,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,32767,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1023,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,32767,0.02994133283694585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,65535,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,65535,0.06020799775918325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,65535,0.051488002141316734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,31,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,255,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,65535,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,511,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,511,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,511,0.012597333639860153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1023,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1023,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1023,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,255,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1023,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,2047,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,2047,0.016783999900023144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,2047,0.016864000509182613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,4095,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,4095,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,4095,0.013359999905029932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,4095,0.01876266673207283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,8191,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,8191,0.025146665672461193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,8191,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,8191,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,16383,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,16383,0.039306665460268654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,16383,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,16383,0.03001066545645396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,32767,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,32767,0.06015466650327047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,32767,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,32767,0.052095999320348106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,65535,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,65535,0.10192533334096272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,3,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,7,0.011781333635250727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,127,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,127,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,255,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,511,0.011989332735538483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,511,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1023,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1023,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,2047,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,2047,0.017157333592573803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,2047,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,2047,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,4095,0.013429333766301474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,65535,0.0867146650950114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,4095,0.023103999594847362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1023,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,4095,0.021397332350413006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,8191,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,8191,0.038106667498747505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,8191,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,8191,0.030106666187445324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,16383,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,65535,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,16383,0.058149332801500954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,16383,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,16383,0.05182399849096934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,32767,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,32767,0.09980799754460652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,32767,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,32767,0.08710400263468425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,65535,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,65535,0.18314133087793985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,65535,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,65535,0.1567626694838206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,31,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,4095,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,511,0.011813333878914515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,511,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1023,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1023,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,2047,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,2047,0.015802666544914246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,4095,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,4095,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,8191,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,8191,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,16383,0.03993066648642222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,16383,0.027493332823117573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,32767,0.06032533446947733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,15,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,63,0.012015999605258306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,32767,0.04423466821511587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,255,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,511,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,511,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1023,0.01340266689658165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,2047,0.01874133323629697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,2047,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,4095,0.021712000171343487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,8191,0.04032533367474874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,8191,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,16383,0.060271998246510826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,16383,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,32767,0.1011306643486023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,4095,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,32767,0.058373332023620605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,3,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,15,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,511,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,2047,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,2047,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,4095,0.038719999293486275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,4095,0.02346666653951009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,8191,0.05867200096448263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,8191,0.039434666434923805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1023,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,16383,0.09940266609191895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,16383,0.05850133299827576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,32767,0.1821333368619283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,3,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,3,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,7,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,7,0.012400000045696894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,15,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,15,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,31,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,31,0.012719999998807907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,63,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,63,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,127,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,127,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,255,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,511,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,511,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1023,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1023,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,2047,0.03937066594759623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,2047,0.023978665471076965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,4095,0.060309335589408875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,4095,0.03999999910593033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,8191,0.10214933753013611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,8191,0.0603359987338384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,255,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,16383,0.18476800123850504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,32767,0.09874666730562846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,32767,0.3510613441467285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,16383,0.10071466366449992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,7,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,511,0.011866666376590729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,511,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,2047,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,2047,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,4095,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,8191,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,8191,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,32767,0.18260266383488974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,16383,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1023,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,32767,0.027274665733178455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,32767,0.028143999477227528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,65535,0.031632001201311745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,65535,0.03143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,15,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,16383,0.02141333371400833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,3,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,131071,0.035360001027584076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,15,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,127,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,131071,0.03550933301448822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,255,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,511,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1023,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1023,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,2047,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,2047,0.012165332833925882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,4095,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,8191,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,8191,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,16383,0.019130667050679524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,511,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,32767,0.023258666197458904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,32767,0.02179733415444692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,4095,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,65535,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,65535,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,3,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,16383,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,131071,0.029626667499542236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,131071,0.027514666318893433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,31,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,255,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,2047,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,4095,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,8191,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,8191,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,16383,0.019482667247454327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,16383,0.01882133384545644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,32767,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,32767,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,65535,0.023381332556406658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,65535,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,4095,0.012608000387748083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,131071,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,131071,0.02733866622050603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,255,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1023,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1023,0.010496000448862711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,2047,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,2047,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,4095,0.01563199982047081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,4095,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,8191,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,8191,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,16383,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,16383,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,32767,0.021375998854637146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,32767,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,65535,0.03790933390458425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,65535,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,511,0.011695999652147293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,131071,0.058277333776156105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,131071,0.03979199876387914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,31,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,255,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,511,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1023,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1023,0.012655999511480331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,2047,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,2047,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,3,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,4095,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,8191,0.01800000046690305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,8191,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,16383,0.02275199939807256
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,16383,0.021344001094500225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,32767,0.025450666745503742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,32767,0.02369066576162974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,65535,0.027029333015282948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,65535,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,15,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,31,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,31,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,63,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,127,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1023,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,2047,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,2047,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,4095,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,8191,0.017429333180189133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,8191,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,16383,0.02011200040578842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,16383,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,32767,0.02181866765022278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,32767,0.019754666835069656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,65535,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,65535,0.022170667846997578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,131071,0.029802667597929638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,131071,0.04304533203442892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,131071,0.02943466603755951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,131071,0.030991998811562855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,31,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,255,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,2047,0.013349333157142004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,4095,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,4095,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,8191,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,511,0.011760000139474869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,16383,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,32767,0.023232000569502514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,32767,0.021338666478792827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,65535,0.0379573330283165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,8191,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,65535,0.02521066615978877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,7,0.013914667069911957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,131071,0.06010133524735769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,131071,0.04037333279848099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,15,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,15,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,16383,0.021055998901526134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1023,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,2047,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,2047,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,4095,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,4095,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,8191,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,8191,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,16383,0.023007998863856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,16383,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,32767,0.037392000357309975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,32767,0.023525332411130268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,255,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,65535,0.056101332108179726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,65535,0.037733333806196846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1023,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,131071,0.0983893374602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,131071,0.05724266668160757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,15,0.01191466674208641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1023,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1023,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,2047,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,2047,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,4095,0.02216533323129018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,4095,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,8191,0.039877332746982574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,8191,0.027477333943049114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,511,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,16383,0.0605973352988561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,3,0.012421333541472753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,7,0.011781333635250727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,31,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,16383,0.04358933369318644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,127,0.01181866725285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,255,0.011829332758982977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,511,0.01357866699496905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1023,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1023,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,2047,0.021738665799299877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,4095,0.03978666663169861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,511,0.012346666306257248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,8191,0.058677335580190025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,8191,0.039349332451820374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,16383,0.10054399569829305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,16383,0.05856533348560333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,3,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,3,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,7,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,2047,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,15,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,15,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,4095,0.023957334458827972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,31,0.012543999900420507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,63,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,63,0.012719999998807907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,127,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,7,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,255,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,255,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,31,0.01358933374285698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,511,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1023,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1023,0.01740266631046931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,2047,0.040133332212766014
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,127,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,2047,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,4095,0.05997333427270254
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,4095,0.04106133431196213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,8191,0.10172266761461894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,8191,0.06022400160630544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,16383,0.18385599056879678
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1,0.014789332946141561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,3,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,7,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,7,0.014175999909639359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,15,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,511,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,31,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,31,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,63,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,16383,0.10147733489672343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,63,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,3,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,127,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,255,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,255,0.0141546664138635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,511,0.021018666525681812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,511,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1023,0.035402665535608925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1023,0.021562665700912476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,2047,0.06028266747792562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,2047,0.03984000037113825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,4095,0.10153599580128987
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,4095,0.06044266621271769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,8191,0.18548800547917685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,15,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,16383,0.18413333098093668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,127,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,3,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,16383,0.353493332862854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,31,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,8191,0.10125333070755005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,511,0.011754666765530905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1023,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,2047,0.014287999520699183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,2047,0.01452800010641416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,4095,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,255,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,4095,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,8191,0.0194560003777345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,8191,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,16383,0.022357332209746044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,16383,0.021269333859284718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,32767,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,65535,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,65535,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,32767,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,131071,0.045797333121299744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,131071,0.03324266771475474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,3,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,7,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,31,0.010543999572594961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,15,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1023,0.01258133351802826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,2047,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,2047,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,4095,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,4095,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,8191,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,8191,0.019808000574509304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,16383,0.019685332973798115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,32767,0.02327466756105423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,32767,0.02146133283774058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,65535,0.04092800120512644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,65535,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,16383,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,131071,0.06102400024731954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,7,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,131071,0.04368533194065094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,255,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,7,0.010426666587591171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,511,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,2047,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,2047,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,4095,0.018826667219400406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,8191,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,8191,0.018757333358128864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,16383,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,16383,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,32767,0.03984000037113825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,32767,0.02499199906984965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,65535,0.06010666489601135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,65535,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,3,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,131071,0.09988266229629517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,131071,0.05916266640027364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1023,0.012351999680201212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1023,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,2047,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,2047,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,4095,0.018325333793958027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,4095,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,8191,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,8191,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,16383,0.03860266755024592
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,16383,0.023717333873112995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,32767,0.05950400233268738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,65535,0.10010133186976115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,65535,0.057445332407951355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,32767,0.03815466662247976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,3,0.013503999759753546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,131071,0.18421866496404013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,3,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,7,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,7,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,15,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,15,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,31,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,63,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,127,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,131071,0.09916266798973083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,31,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1023,0.015493333339691162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,2047,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,4095,0.0399893323580424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,4095,0.02532266577084859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,8191,0.060778667529424034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1,0.013359999905029932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,2047,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,3,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,3,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,7,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,7,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,15,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,15,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,31,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,31,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,63,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,63,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,127,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,8191,0.041936000188191734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,255,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,255,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,511,0.015509333461523056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,511,0.013957332819700241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1023,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1023,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,2047,0.04048533240954081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,2047,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,4095,0.06215466558933258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,4095,0.041536000867684685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,127,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,8191,0.06163733204205831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,3,0.015354666858911514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,3,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,7,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,7,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,15,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,255,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,15,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,31,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,31,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,63,0.014581333845853806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,127,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,127,0.01469333345691363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,255,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,255,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,511,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,511,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1023,0.03554133325815201
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1023,0.021744000415007275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,2047,0.06200533111890157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,2047,0.04127999891837438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,8191,0.1023573378721873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,63,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,4095,0.062047998110453285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1,0.01952533299724261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,8191,0.18447466691335043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,8191,0.1032960017522176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,3,0.019402666638294857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,3,0.017504000415404636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,7,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,7,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,15,0.01913600042462349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,15,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,31,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,31,0.018687999496857326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,63,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,63,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,127,0.019472000499566395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,127,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,255,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,255,0.017477333545684814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,511,0.03350933392842611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,511,0.023237332701683044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,4095,0.10168000062306722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1023,0.052943999568621315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1023,0.03518400092919668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,2047,0.09508267045021057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,4095,0.1729066570599874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,4095,0.094842662413915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,8191,0.32531734307607013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,8191,0.16666133205095926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1,0.013573333621025085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,3,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,3,0.013904000322024027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,7,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,7,0.013877333452304205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,15,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,15,0.013594667116800943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,31,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,31,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,63,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,63,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,127,0.01339200014869372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,127,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,255,0.013381333400805792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,2047,0.05811200042565664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,511,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,511,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1023,0.019733333339293797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1023,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,2047,0.04035733391841253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,2047,0.02769600103298823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,3,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,255,0.01320533330241839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,3,0.015749332805474598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,7,0.015482666591803232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,15,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,15,0.015365333606799444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,31,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,63,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,63,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,127,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,127,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,255,0.016229332735141117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,255,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,511,0.020256000260512035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1023,0.03575466573238373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1023,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,2047,0.06230400005976359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,7,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,31,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1,0.017024000485738117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,3,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,3,0.019066666563351948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,7,0.021594665944576263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,7,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,15,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,15,0.019002666076024372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,31,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,511,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,63,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,63,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,127,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,127,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,255,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,255,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,511,0.03331733246644338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,511,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1023,0.0537066658337911
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1023,0.03384533276160558
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,31,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,2047,0.09650133053461711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,2047,0.05799466868241628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1,0.025663999219735462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,3,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,3,0.025445332129796345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,7,0.02942399928967158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,7,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,15,0.02924266705910365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,15,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,31,0.029258665939172108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,2047,0.041690667470296226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,63,0.02958933264017105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1,0.029701332251230877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,127,0.029189333319664
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,127,0.02508266766866048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,255,0.032645332316557564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,255,0.025578667720158894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,31,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,511,0.05409599840641022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,63,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,511,0.037791999677817024
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1023,0.09117866555849712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1023,0.0561653325955073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,2047,0.17319466670354208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,2047,0.0972160001595815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1,0.018816000471512478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1,0.01710933322707812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,3,0.017722666263580322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,3,0.01720000058412552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,7,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,7,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,15,0.01884799947341283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,15,0.016810666769742966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,31,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,63,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,127,0.017594666530688603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,127,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,255,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,511,0.023039999107519787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,63,0.019610666980346043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,511,0.02004266654451688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,255,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1023,0.03742400060097376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,3,0.02082666630546252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,3,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,7,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,7,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,15,0.021087999145189922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1023,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,31,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,31,0.019658666104078293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,31,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,63,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,127,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,127,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,255,0.021370666722456615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,15,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,511,0.03448000053564707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,63,0.020928000410397846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1023,0.055029332637786865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1023,0.034143999218940735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1,0.0296426663796107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1,0.02554133286078771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,3,0.029829333225886028
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,3,0.02717866748571396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,7,0.029637334247430164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,7,0.02683199942111969
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,15,0.029898665845394135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,15,0.026101333399613697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,31,0.03019733230272929
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,31,0.026602665583292644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,63,0.030117332935333252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,255,0.019589333484570186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,63,0.027280000348885853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,127,0.031178665657838184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,127,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,255,0.034373333056767784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,255,0.02722666660944621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,511,0.05418133238951365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,511,0.037471999724706016
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1023,0.09336533149083455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1,0.04789866507053375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1,0.0417546679576238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,3,0.04826133449872335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,511,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,7,0.04761599997679392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,7,0.04170133173465729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,15,0.04761599997679392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,15,0.04166933397452036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,31,0.0476693312327067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1023,0.056074668963750206
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,31,0.041840001940727234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,63,0.04817600051561991
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,63,0.04167999823888143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,127,0.05009066561857859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,127,0.03979733337958654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,255,0.05420266588528951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,255,0.041834667325019836
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,511,0.09292800227801006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,511,0.06033066908518473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1023,0.16796799500783285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,7,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,3,0.04172799984614054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1023,0.09516266981760661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,127,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,255,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,511,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1023,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1023,0.012671999633312225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,2047,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,2047,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,4095,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,4095,0.017551999539136887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,8191,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,8191,0.018746666610240936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,16383,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,16383,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,32767,0.025258667767047882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,32767,0.023792001108328503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,65535,0.02789866675933202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,131071,0.06419733166694641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,3,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,7,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,15,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,15,0.01051733394463857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,31,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,63,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,131071,0.04568000137805939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,65535,0.041797334949175514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,255,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,511,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,511,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1023,0.011952000359694162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,2047,0.016810666769742966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,2047,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,4095,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,8191,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,8191,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,16383,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,16383,0.020981334149837494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,32767,0.04055466751257578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,32767,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,511,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,65535,0.05971199770768484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,65535,0.040234667559464775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,7,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,131071,0.0995199978351593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,7,0.0122079998254776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,4095,0.017765333255132038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,15,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,255,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,511,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,131071,0.05904533465703329
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1023,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,2047,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,2047,0.015317333241303762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,4095,0.017152000218629837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,4095,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,8191,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,8191,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1023,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,16383,0.02369600037733714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,32767,0.05866666634877523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,32767,0.03828266759713491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,511,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,65535,0.09963200489679973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,65535,0.05797333518664042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,131071,0.18294399976730347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,16383,0.03886399914820989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,511,0.011514666179815928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,131071,0.09963200489679973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1023,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,2047,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,4095,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,4095,0.019434666881958645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,8191,0.03843733419974645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,8191,0.02316266546646754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,16383,0.05952000121275584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,2047,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,32767,0.10069333513577779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,65535,0.18268267313639322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,16383,0.03750933210055033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,65535,0.0976639986038208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1023,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,131071,0.35014931360880536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,131071,0.17899733781814575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,32767,0.05881066620349884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,1,0.021397332350413006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,3,0.026736001173655193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,3,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,7,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,7,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,15,0.025536000728607178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,15,0.021695998807748158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,1,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,31,0.025093334416548412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,31,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,63,0.02548266698916753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,63,0.022991999983787537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,127,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,127,0.021386665602525074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,255,0.025055999557177227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,255,0.023413332800070446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,511,0.03751999884843826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,511,0.02752533306678136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,1,0.02940800040960312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,3,0.03156800071398417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,3,0.028570666909217834
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,7,0.03179199993610382
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,7,0.028858666618665058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,15,0.03142400085926056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,15,0.02922133356332779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,31,0.031301334500312805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,31,0.028858666618665058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,63,0.03226666649182638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,63,0.029135999580224354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,127,0.03209600100914637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,127,0.029205332199732464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,255,0.0373333344856898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,255,0.027679999669392902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,1,0.03147733211517334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,511,0.058058664202690125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,511,0.039818666875362396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,1,0.050106664498647056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,1,0.04385599990685781
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,3,0.05039466420809428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,3,0.04402133325735728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,7,0.050341332952181496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,7,0.04367466767628988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,15,0.04986133178075155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,15,0.04394133388996124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,31,0.0499893327554067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,31,0.042917331059773765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,63,0.0503359983364741
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,63,0.04418133199214935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,127,0.05328000088532766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,127,0.04233066737651825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,255,0.05630399783452352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,255,0.04620266457398733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,511,0.09498133261998494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,511,0.06237866481145223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,1,0.08892266949017842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,1,0.07250666618347168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,3,0.08906132976214091
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,3,0.07229866584142049
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,7,0.08885332942008972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,7,0.0726506660381953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,15,0.08904000123341878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,15,0.07284800211588542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,31,0.08932800094286601
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,31,0.07229333122571309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,63,0.08898666501045227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,63,0.07257066667079926
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,127,0.09051199754079182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,127,0.07397333284219106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,255,0.09504533807436626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,511,0.17266666889190674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,511,0.10973866780598958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,1,0.03988266736268997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,1,0.03473600000143051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,3,0.03953066716591517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,255,0.07542933523654938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,3,0.03409066547950109
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,7,0.03568000098069509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,15,0.03991466760635376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,15,0.035445332527160645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,7,0.040037333965301514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,31,0.03993066648642222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,31,0.03549866626660029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,63,0.03955733279387156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,63,0.035216001172860466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,127,0.03443733354409536
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,127,0.041840001940727234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,255,0.042277331153551735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,255,0.03573333223660787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,1,0.054005334774653115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,1,0.04784533381462097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,3,0.05406400064627329
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,3,0.04731200138727824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,7,0.05434666574001312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,7,0.047728002071380615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,15,0.05416533350944519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,15,0.04645333190759023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,31,0.054117331902186074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,31,0.04764266808827718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,63,0.053914666175842285
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,63,0.048101335763931274
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,127,0.05632533133029938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,127,0.04795200129350027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,255,0.06158400078614553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,255,0.05096533397833506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,1,0.09103467067082723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,1,0.07891199986139934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,3,0.09217066566149394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,3,0.07851733267307281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,7,0.07897600034872691
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,15,0.09114666779836018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,15,0.07797866563002269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,31,0.09145067135492961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,31,0.07715199887752533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,63,0.0922933320204417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,63,0.07840000092983246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,127,0.09307733178138733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,127,0.0786240001519521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,255,0.10013332962989807
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,7,0.09213866790135701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,1,0.13827733198801676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,3,0.16486933827400208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,255,0.08122666676839192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,3,0.13826133807500204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,7,0.16473066806793213
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,7,0.13800533612569174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,15,0.16474666198094687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,15,0.1379146675268809
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,31,0.1648373305797577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,31,0.13800000150998434
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,1,0.16521599888801575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,63,0.16482133666674295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,127,0.16686934232711792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,127,0.13749333222707114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,255,0.1776319940884908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,3,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,255,0.14036267002423605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,7,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,7,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,15,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,31,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,63,0.011567999919255575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,511,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1023,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,2047,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,2047,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,4095,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,4095,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,8191,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,8191,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,16383,0.024517332514127094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,16383,0.023290666441122692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,32767,0.04173333446184794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,32767,0.027461332579453785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,63,0.13819733262062073
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,65535,0.06125866870085398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,3,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,15,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,31,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,63,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,127,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,255,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,511,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,511,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1023,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,2047,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,2047,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,4095,0.01972266659140587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,4095,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,8191,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,8191,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,16383,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,16383,0.025173333783944447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,32767,0.05986666679382324
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,32767,0.03966933240493139
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,65535,0.04363200068473816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,65535,0.06029866635799408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,15,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,31,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,255,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,511,0.012671999633312225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,511,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1023,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1023,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,2047,0.01756799966096878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,2047,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,4095,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,4095,0.019167999426523846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,8191,0.03809600075085958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,8191,0.023584000766277313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,16383,0.059631998340288796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,65535,0.0995146632194519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,32767,0.09973866740862529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,32767,0.05805333455403646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,65535,0.1807360053062439
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,3,0.012719999998807907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,65535,0.09810133775075276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,15,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,63,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,127,0.011658667276302973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,255,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,7,0.01180800050497055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,511,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1023,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1023,0.013445333888133367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,2047,0.019445333629846573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,2047,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,4095,0.0379573330283165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,4095,0.02348800003528595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,8191,0.05825600028038025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,8191,0.03762666632731756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,16383,0.09845866759618123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,16383,0.05795733133951823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,32767,0.1811573306719462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,16383,0.039642666776975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,65535,0.34701867898305255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,32767,0.09760000308354695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,3,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,7,0.010501333822806677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,511,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,511,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1023,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1023,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,2047,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,2047,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,2047,0.01349866638580958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,2047,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,65535,0.17929067214330038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,4095,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,4095,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,8191,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,8191,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,8191,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,8191,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,4095,0.015669333438078564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,16383,0.02332799881696701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,16383,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,16383,0.02128533273935318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,4095,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,32767,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,32767,0.039647998909155525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,32767,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,32767,0.03164800008138021
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,65535,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,65535,0.06070399781068166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,65535,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1,0.012682666381200155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,15,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,65535,0.05208000044027964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,15,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,16383,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,127,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,511,0.011866666376590729
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,511,0.011989332735538483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1023,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1023,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,2047,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,2047,0.013541333377361298
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,2047,0.015477333217859268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,4095,0.013738666971524557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,4095,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,4095,0.014042666802803675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,4095,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,8191,0.024160000185171764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,8191,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,8191,0.02183466653029124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,16383,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,8191,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,16383,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,16383,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,16383,0.030634666482607525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,32767,0.013850666582584381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,32767,0.06010133524735769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,32767,0.01458666721979777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,32767,0.05204799771308899
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,65535,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,65535,0.10250666737556458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,65535,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,2047,0.01691199963291486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,3,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,15,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,31,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,255,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,63,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,65535,0.08758933345476787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,511,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,511,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,511,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,511,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1023,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,2047,0.01844266677896182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,2047,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,2047,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,4095,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,4095,0.013327999661366144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,4095,0.021701333423455555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,8191,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,8191,0.03752533346414566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,8191,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,8191,0.02958933264017105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,16383,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,16383,0.058245331048965454
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,16383,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,16383,0.05186666548252106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1023,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,32767,0.10095999638239543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,32767,0.014778666198253632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,32767,0.08682133754094441
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1,0.010053333515922228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,65535,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1,0.009717333440979322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,65535,0.1567573348681132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,7,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,15,0.009813333551088968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,65535,0.1840426723162333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,63,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,63,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,127,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,255,0.010469333579142889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,511,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,511,0.011621333658695221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,511,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1023,0.011567999919255575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,2047,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,2047,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,2047,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,4095,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,4095,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,4095,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,8191,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,8191,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1023,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,8191,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,16383,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,16383,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,16383,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,16383,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,32767,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,32767,0.027461332579453785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,32767,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,32767,0.02737066646416982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,8191,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,65535,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,2047,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,65535,0.02962133288383484
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,131071,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,65535,0.028880000114440918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,131071,0.03365866591533025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,65535,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,3,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,7,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,7,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,15,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,31,0.0102613332370917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,63,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,63,0.010048000141978264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,127,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,131071,0.031717332700888314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,255,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,511,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,511,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,2047,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,2047,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,2047,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,2047,0.01198400060335795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,4095,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,4095,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,4095,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,8191,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,8191,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,8191,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,131071,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,16383,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,16383,0.019130667050679524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,16383,0.01899733394384384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,32767,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,32767,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,32767,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,16383,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,32767,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,65535,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,65535,0.02274666726589203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,8191,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,65535,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,65535,0.023029332359631855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,131071,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,131071,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,15,0.009685333197315535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,15,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,63,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,63,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,127,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,127,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,255,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,511,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,511,0.010570666442314783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,511,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1023,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1023,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1023,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,2047,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,2047,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,2047,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,2047,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,4095,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,4095,0.012266666938861212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,131071,0.025493333737055462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,4095,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,8191,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,8191,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,8191,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,8191,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,16383,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,16383,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,16383,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,16383,0.017530667285124462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,131071,0.027434666951497395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,32767,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,32767,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,32767,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,4095,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,65535,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,65535,0.021701333423455555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,65535,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,65535,0.019386666516462963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,32767,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,131071,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,131071,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,3,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,3,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,131071,0.027285332481066387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,7,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,15,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,131071,0.02733866622050603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,15,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,31,0.012351999680201212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,255,0.0116799995303154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,511,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1023,0.011594666788975397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,31,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,2047,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,2047,0.011706666400035223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,2047,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,4095,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,4095,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,4095,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,4095,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,8191,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,8191,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1023,0.011450666934251785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,8191,0.016821333517630894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,16383,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,16383,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,16383,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,16383,0.020608000457286835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,32767,0.011535999675591787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,32767,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,32767,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,32767,0.02310933421055476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,8191,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,65535,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,65535,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,65535,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,65535,0.02310933421055476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,131071,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,131071,0.027621333797772724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,2047,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,3,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,131071,0.027503999571005504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,3,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,7,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,15,0.010480000327030817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,15,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,131071,0.012650666137536367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,3,0.010570666442314783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,511,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,511,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1023,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1023,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,2047,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,2047,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,2047,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,4095,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,4095,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,4095,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,4095,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,8191,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,8191,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,2047,0.01267733300725619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,16383,0.012719999998807907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,16383,0.019498666127522785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,16383,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,8191,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,16383,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,32767,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,32767,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,8191,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,32767,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,32767,0.01939733326435089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,65535,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,65535,0.02219199885924657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,65535,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,65535,0.021322667598724365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,131071,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,131071,0.02792533238728841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,131071,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,7,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,7,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,7,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,15,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,131071,0.0271519993742307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,31,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,31,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,127,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,255,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,511,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1023,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,2047,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,2047,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,2047,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,2047,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,4095,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,4095,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,4095,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,8191,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,8191,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,8191,0.015504000087579092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,16383,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,16383,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,16383,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,16383,0.016885332763195038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,32767,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,32767,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,127,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,32767,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,32767,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,8191,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,65535,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,65535,0.02510400116443634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,65535,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,65535,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,131071,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,131071,0.039893334110577904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,3,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,7,0.010618666807810465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,131071,0.031888000667095184
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,15,0.010485333700974783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,15,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,31,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,131071,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,7,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,63,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,127,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,127,0.011829332758982977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,127,0.011658667276302973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,255,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,511,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,511,0.012714666624863943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1023,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1023,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1023,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,2047,0.017103999853134155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,2047,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,2047,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,4095,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,4095,0.018885333091020584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,4095,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,4095,0.01871466636657715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,8191,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,8191,0.023498666783173878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,8191,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,8191,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,16383,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,16383,0.03976533313592275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,16383,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,16383,0.03183999905983607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,32767,0.014538666854302088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,32767,0.060309335589408875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,32767,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,7,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,127,0.011551999797423681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,32767,0.05273599922657013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,511,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,511,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1023,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1023,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1023,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,2047,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,2047,0.01842133328318596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,2047,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,2047,0.017445333302021027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,4095,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,4095,0.02334933231274287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,4095,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,8191,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,8191,0.0397119993964831
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,8191,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,4095,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,16383,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,16383,0.06042666733264923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,16383,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,8191,0.03145600110292435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,16383,0.053488001227378845
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,32767,0.10125866532325745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,32767,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,3,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,3,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,3,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,7,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,15,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,31,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,31,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,32767,0.08694400389989217
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,63,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,63,0.012330666184425354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,127,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,7,0.01358933374285698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,127,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,127,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,255,0.01369599997997284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,255,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,255,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,511,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,127,0.012703999876976013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1023,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1023,0.01681600014368693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1023,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1023,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,2047,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,2047,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,2047,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,2047,0.023183998962243397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,4095,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,4095,0.040250666439533234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,4095,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,4095,0.031114667654037476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,8191,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,255,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,8191,0.015370666980743408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,8191,0.05226666728655497
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,16383,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,16383,0.10334933797518413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,16383,0.015370666980743408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,8191,0.05995733539263407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,32767,0.015295999745527903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,32767,0.18661334117253622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,16383,0.08898666501045227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,511,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,32767,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,32767,0.15842666228612265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,31,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,15,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,255,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,255,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,7,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,63,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,511,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1023,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1023,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,2047,0.013424000392357508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,2047,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,2047,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,4095,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,4095,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,4095,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,4095,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,8191,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,8191,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,8191,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,16383,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,16383,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,16383,0.019674666225910187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,8191,0.013349333157142004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,16383,0.01950399950146675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,32767,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,32767,0.02146133283774058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,32767,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,65535,0.023418667415777843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,32767,0.021194666624069214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,65535,0.012757333616415659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,65535,0.021615999440352123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,131071,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,131071,0.029301332930723827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,131071,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,131071,0.02741866558790207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,3,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,3,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,7,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,15,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,31,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,63,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,63,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,127,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,255,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,255,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,511,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1023,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,2047,0.01339200014869372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,2047,0.01333333303531011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,4095,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,511,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,4095,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,4095,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,8191,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,2047,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,2047,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,8191,0.013354666531085968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,8191,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,16383,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,16383,0.01959466685851415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,16383,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,4095,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,16383,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,32767,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,32767,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,32767,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,65535,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,65535,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,65535,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,32767,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,8191,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,65535,0.023402666052182514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,131071,0.013951999445756277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,131071,0.04351999859015147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,131071,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,3,0.010581333190202713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,131071,0.031717332700888314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,3,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,7,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,15,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,31,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,63,0.010399999717871347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,255,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,511,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,511,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,511,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,511,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1023,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1023,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,2047,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,2047,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,2047,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,4095,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,4095,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,4095,0.016869333883126576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,8191,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,8191,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,8191,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,8191,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,16383,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,16383,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,16383,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,16383,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,32767,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,32767,0.02515200028816859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,32767,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,32767,0.023621333142121632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,65535,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,65535,0.03895466774702072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,65535,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,65535,0.031311998764673867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,131071,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,131071,0.05975466469923655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,131071,0.05292266607284546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1,0.011514666179815928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,7,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,15,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,15,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,31,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,31,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,511,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,511,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1023,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,131071,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,2047,0.014352000008026758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,2047,0.014058666924635569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,2047,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,4095,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,4095,0.023247999449570973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,2047,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,8191,0.014287999520699183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,8191,0.03745600084463755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,8191,0.01413333291808764
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,8191,0.03086400032043457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,4095,0.023061332603295643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,16383,0.014170666535695394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,16383,0.05935466786225637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,16383,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,16383,0.05320533116658529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1,0.012645332763592402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,4095,0.014762666076421738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,3,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,3,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,7,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,7,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,7,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,15,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,3,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,15,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,31,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,31,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,63,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,63,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,63,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,127,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,127,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,127,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,15,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,255,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,511,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,511,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,511,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1023,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1023,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1023,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,2047,0.01552533358335495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,255,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,3,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,2047,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,2047,0.02295999974012375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,4095,0.015552000453074774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,4095,0.03995199998219808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,4095,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,4095,0.03148799886306127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,8191,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,8191,0.061205332477887474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,8191,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,2047,0.0232640008131663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,8191,0.054287999868392944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,16383,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,16383,0.10308266679445903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,16383,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,3,0.01339200014869372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,3,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,3,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,3,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,7,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,7,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,7,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,7,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,15,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,15,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,16383,0.08896000186602275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,31,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,31,0.014208000153303146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,31,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,31,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,63,0.013562666873137156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,15,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,63,0.013936000565687815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,15,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,63,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,127,0.013482666263977686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,127,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,127,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,255,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,255,0.01552533358335495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,255,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,255,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,511,0.013290667285521826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,511,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,511,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,63,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1023,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1023,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1023,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,2047,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,2047,0.040207999447981514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,2047,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,2047,0.03143999973932902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,4095,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,4095,0.06233599781990051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,4095,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,4095,0.0545653353134791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,8191,0.016901332885026932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,8191,0.017466666797796886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,8191,0.08896533648173015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,16383,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1023,0.023381332556406658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,16383,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,3,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,3,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,3,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,16383,0.1590026617050171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,7,0.011957333733638128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,8191,0.10362666845321655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,7,0.012186666329701742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,7,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,15,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,15,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,3,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,15,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,15,0.012719999998807907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,31,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,16383,0.18730666240056357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,7,0.012426666915416718
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,63,0.012154666086037954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,63,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,63,0.012341332932313284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,127,0.012122667084137598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,31,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,127,0.012223999947309494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,127,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,31,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,127,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,255,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,255,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,255,0.013295999417702356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,255,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,511,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,511,0.014175999909639359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,511,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,511,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1023,0.014394666999578476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1023,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1023,0.016085332880417507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,2047,0.02367999901374181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,2047,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,2047,0.022863999009132385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,4095,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,4095,0.04061333338419596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,4095,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,3,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,3,0.01469333345691363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,3,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,7,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,7,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,7,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,7,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,2047,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,15,0.015471999843915304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,15,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,15,0.015381333728631338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,31,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,4095,0.03126933425664902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,31,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,15,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,31,0.01340266689658165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,63,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,63,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,63,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,63,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,127,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,127,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,127,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,255,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,255,0.013658666362365087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,255,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,31,0.013408000270525614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,511,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,511,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,511,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,511,0.016895999511082966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1023,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1023,0.02218666672706604
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,127,0.013717333475748697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1023,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1023,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,255,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,2047,0.03977599988381068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,2047,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,2047,0.03160533308982849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,4095,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,4095,0.06223999957243601
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,4095,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,4095,0.053258667389551796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,3,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,3,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,3,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,7,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,2047,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,7,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,15,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,15,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,7,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,15,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,7,0.017792000124851864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,31,0.016938666502634685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,31,0.01793066660563151
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,31,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,31,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,63,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,63,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,63,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,63,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,127,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,127,0.017445333302021027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,3,0.01754666616519292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,15,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,127,0.017136000096797943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,255,0.01883200059334437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,255,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,255,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,255,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,511,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,511,0.023530667026837666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,511,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,511,0.02149333308140437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1023,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1023,0.035071998834609985
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1023,0.02956799914439519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,127,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,2047,0.021231998999913532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,2047,0.05850133299827576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,2047,0.02086399992307027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,2047,0.05128000179926554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,4095,0.021541332205136616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,4095,0.09542399644851685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,4095,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1023,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1,0.0144213338692983
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,4095,0.08282666901747386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,3,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,3,0.015087999403476715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,3,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,7,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,7,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,7,0.01481066644191742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,15,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,15,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1,0.01458666721979777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,15,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,31,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,31,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,31,0.014325333138306936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,31,0.014789332946141561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,63,0.01471466695268949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,63,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,63,0.013754667093356451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,63,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,127,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,127,0.01504533365368843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,127,0.013839999834696451
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,127,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,255,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,255,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,15,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,255,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,255,0.014864000181357065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,511,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,511,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1023,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1023,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1023,0.015082667271296183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1023,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,511,0.015471999843915304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,2047,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,2047,0.03997333347797394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,2047,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,2047,0.03147733211517334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1,0.01727466657757759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,3,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,3,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1,0.01883200059334437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,3,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,3,0.017450666675964992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,7,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,511,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,7,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,7,0.017674667139848072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,15,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,15,0.018592000007629395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,15,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,15,0.01720533271630605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1,0.016858667135238647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,31,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,31,0.017407999684413273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,31,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,31,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,63,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,63,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,63,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,63,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,127,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,127,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,127,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,127,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,255,0.019253333409627277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,255,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,7,0.018309333672126133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,255,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,511,0.019050666441520054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,511,0.0234400009115537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,511,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,511,0.021301334102948506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1023,0.019130667050679524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1023,0.034688000877698265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1023,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1023,0.029285334050655365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,255,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,2047,0.021141332884629566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,2047,0.05880533158779144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,2047,0.05057600140571594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1,0.023567999402681988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,3,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,3,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,3,0.0236160010099411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,3,0.02380266785621643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,7,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,2047,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,7,0.02367466688156128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1,0.023157333334287006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,7,0.02333866556485494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,15,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,15,0.025125332176685333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,15,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,15,0.023402666052182514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,31,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,31,0.025461333493391674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,31,0.02370133250951767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,31,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,63,0.02554133286078771
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,63,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,63,0.02386666586001714
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,63,0.025349333882331848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,127,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,127,0.02499199906984965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,127,0.02517866591612498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,127,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,255,0.02735466758410136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,255,0.025546667476495106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,7,0.025546667476495106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,255,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,511,0.025253333151340485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,511,0.03760000069936117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,511,0.02329600105683009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,511,0.031471999982992806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1023,0.05618133147557577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1023,0.023386667172114056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1023,0.05089599887530009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,255,0.025472000241279602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,2047,0.029338667790095013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,2047,0.09841600060462952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1023,0.0258240004380544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,3,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,15,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,3,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,15,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,31,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,31,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,63,0.01044800008336703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,2047,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,2047,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,4095,0.01526933287580808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,4095,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,4095,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,8191,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,8191,0.017562666287024815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,8191,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,8191,0.017269333203633625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,16383,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,16383,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,16383,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,16383,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,2047,0.02773866554101308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,2047,0.08488000432650249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,32767,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,32767,0.020810666183630627
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,65535,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,65535,0.02537599951028824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,65535,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,65535,0.024821333587169647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,32767,0.02141333371400833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,131071,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,131071,0.041637333730856575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,131071,0.03332266708215078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,3,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,7,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,7,0.012661332885424295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,7,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,15,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,131071,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,127,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,127,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,511,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,511,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1023,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1023,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1023,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,2047,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,2047,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,4095,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,4095,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,8191,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,8191,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,8191,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,8191,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,16383,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,16383,0.02124800036350886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,16383,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,16383,0.0201706662774086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,32767,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,32767,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,255,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,32767,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,32767,0.023605334262053173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,65535,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,65535,0.03989866624275843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,65535,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,65535,0.031343999008337654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,131071,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,131071,0.06006399790445963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,2047,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,3,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,3,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,7,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,7,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,131071,0.01525866612792015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,31,0.010527999450763067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,131071,0.052149335543314614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,63,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,511,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,31,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1023,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1023,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,2047,0.013007999708255133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,2047,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,2047,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,2047,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,4095,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,4095,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,4095,0.012693333129088083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,4095,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,8191,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,8191,0.01870399961868922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,8191,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,8191,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,16383,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,16383,0.02345066765944163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,16383,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,16383,0.02163200080394745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,15,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,32767,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,32767,0.039306665460268654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,32767,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,32767,0.0312266672650973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,65535,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,65535,0.06015466650327047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,65535,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,65535,0.05194666484991709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,131071,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,131071,0.10319466392199199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,131071,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1,0.019167999426523846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,131071,0.08869866530100505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1,0.017727999637524288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,3,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,3,0.018842666099468868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,3,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1,0.019567999988794327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,7,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,7,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,7,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,7,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,15,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,15,0.017616000026464462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,15,0.017456000049908955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,15,0.016970666746298473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,3,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,31,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,31,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,31,0.017125333348910015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,31,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,63,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,63,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,63,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,63,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,127,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,127,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,127,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,127,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,255,0.01961600035429001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,255,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,255,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,255,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,511,0.019237333287795384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,511,0.023306667804718018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,511,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,511,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1023,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1023,0.034661332766215004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1023,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1023,0.029690665503342945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1,0.025536000728607178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1,0.025578667720158894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1,0.025290665527184803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,3,0.02573866645495097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,3,0.025754667818546295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,3,0.02515733242034912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,3,0.025258667767047882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,7,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,7,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,7,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,7,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,15,0.025733334322770435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,15,0.025536000728607178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,15,0.024256000916163128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,15,0.02510400116443634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,31,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,31,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,31,0.02347733328739802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,31,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,63,0.025631998976071674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,63,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,63,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,63,0.02532266577084859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,127,0.025616000096003216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,127,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,127,0.023157333334287006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,127,0.023984000086784363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,255,0.026778665681680042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,255,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,255,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,255,0.023584000766277313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,511,0.027189334233601887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,511,0.03755199909210205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,511,0.031471999982992806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1023,0.027552001178264618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1023,0.05612266560395559
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1023,0.025290665527184803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,511,0.02516266703605652
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1,0.04005333284536997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1023,0.050288001696268715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1,0.039994666973749794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1,0.03790933390458425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,3,0.03994666785001755
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,3,0.04165866722663244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,3,0.037605332831541695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,3,0.038165333370367684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1,0.03915199885765711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,7,0.039701332648595176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,7,0.04145599901676178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,7,0.03782933453718821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,15,0.04167999823888143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,15,0.03739733248949051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,15,0.03770133356253306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,31,0.03973866750796636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,31,0.039808000127474465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,7,0.0373279998699824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,31,0.03759466608365377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,31,0.03756266583998998
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,63,0.03985599925120672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,63,0.0412266676624616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,63,0.037808001041412354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,63,0.03793599953254064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,127,0.039706667264302574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,127,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,127,0.037445334096749626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,127,0.037248000502586365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,255,0.04190933207670847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,255,0.042064001162846885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,255,0.03751999884843826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,255,0.037776000797748566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,511,0.04194133480389913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,15,0.04200533529122671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,511,0.0395359992980957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,511,0.05409066875775655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1023,0.04370133578777313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1023,0.09796266754468282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1023,0.03757333258787791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,511,0.06211733321348826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1023,0.08523199955622356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,1,0.02703999976317088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,1,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,1,0.02516799916823705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,1,0.025557334224383037
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,3,0.027210667729377747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,3,0.025663999219735462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,3,0.02548266698916753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,3,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,7,0.02735999971628189
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,7,0.027269333600997925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,7,0.025237334271272022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,7,0.025434667865435284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,15,0.026687999566396076
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,15,0.027221334477265675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,15,0.025621332228183746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,15,0.02552533398071925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,31,0.02672533442576726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,31,0.02720533311367035
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,31,0.025306666890780132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,31,0.02510400116443634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,63,0.02740799884001414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,63,0.02736533433198929
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,63,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,63,0.02536533276240031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,127,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,127,0.027141332626342773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,127,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,127,0.025519999365011852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,255,0.02759466568628947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,255,0.025727999707063038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,255,0.025498665869235992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,511,0.027690666417280834
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,511,0.03814399987459183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,511,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,1,0.04221333563327789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,1,0.041519999504089355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,1,0.03815466662247976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,255,0.02679466704527537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,3,0.041663999358812966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,3,0.04194133480389913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,3,0.03774933268626531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,3,0.03788800040880839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,511,0.034602666894594826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,7,0.04171200096607208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,1,0.038149334490299225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,7,0.04171733558177948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,7,0.03861333429813385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,7,0.0377813329299291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,15,0.04182399809360504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,15,0.04165333261092504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,15,0.03851733356714249
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,15,0.03799466788768768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,31,0.04154666761557261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,31,0.04161600023508072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,31,0.03811733424663544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,31,0.03770666569471359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,63,0.04161066561937332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,63,0.041696002086003624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,63,0.03800000001986822
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,63,0.037978666524092354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,127,0.04019733270009359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,127,0.04045866678158442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,127,0.037717332442601524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,127,0.03791466603676478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,255,0.043749332427978516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,255,0.04373333354791006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,255,0.03979733337958654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,255,0.037871999045213066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,511,0.04294399917125702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,511,0.03988266736268997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,511,0.05584533512592316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,1,0.07160000006357829
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,1,0.07100800176461537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,1,0.06646933158238728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,1,0.06418133278687795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,3,0.07275733351707458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,3,0.07249066730340321
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,3,0.06489600241184235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,511,0.062165334820747375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,3,0.0647626668214798
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,7,0.0724533349275589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,7,0.07211733361085255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,7,0.06442133088906606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,7,0.06458666423956554
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,15,0.07218666871388753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,15,0.07163733243942261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,15,0.06410133341948192
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,15,0.06514133512973785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,31,0.07243200143178304
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,31,0.07197866837183635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,31,0.06439466774463654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,63,0.07259200016657512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,63,0.07223999996980031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,63,0.06422933439413707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,63,0.0646666685740153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,31,0.06531199812889099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,127,0.07242666681607564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,127,0.07238399982452393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,127,0.06465599934260051
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,127,0.06457599997520447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,255,0.07502933343251546
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,255,0.07260266443093617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,255,0.0668213317791621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,255,0.06842666864395142
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,511,0.07495999832948048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,511,0.06666133304437001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,511,0.09726933638254802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,3,0.010570666442314783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,3,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,7,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,511,0.11133333047231038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,15,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,31,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,31,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,7,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,63,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,511,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1023,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,2047,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,2047,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,2047,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,2047,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,4095,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,4095,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,4095,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,8191,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,8191,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,8191,0.013354666531085968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,8191,0.018559999763965607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,255,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,16383,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,16383,0.019632000476121902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,16383,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,4095,0.01693333312869072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,16383,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,32767,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,32767,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,32767,0.024586667617162068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,65535,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,65535,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,32767,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,65535,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,65535,0.03181333343187968
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,131071,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,131071,0.06001600126425425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,131071,0.052298665046691895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,3,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,131071,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,31,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,63,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,63,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,15,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,127,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,127,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,31,0.010512000570694605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1023,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1023,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1023,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,2047,0.014720000326633453
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,2047,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,2047,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,4095,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,4095,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,4095,0.01544533297419548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,8191,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,8191,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,8191,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,8191,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,16383,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,16383,0.023370665808518726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,16383,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,16383,0.022965334355831146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,32767,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,32767,0.03952533255020777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,32767,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,32767,0.029631999631722767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,65535,0.013306666165590286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,65535,0.06042666733264923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,65535,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,65535,0.052000001072883606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,131071,0.01532799998919169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,131071,0.10483733812967937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,131071,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,131071,0.08893332878748576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,3,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,7,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,7,0.010458666831254959
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,15,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,31,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,63,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,15,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,127,0.010496000448862711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,255,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,255,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,511,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1023,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1023,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1023,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,2047,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,2047,0.01687466725707054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,2047,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,511,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,511,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,4095,0.019637333850065868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,4095,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,4095,0.017685333887736004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,8191,0.013466666142145792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,8191,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,2047,0.015989333391189575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,8191,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,4095,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,8191,0.0232640008131663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,16383,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,16383,0.038634667793909706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,16383,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,16383,0.02998399982849757
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,32767,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,32767,0.0595413347085317
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,32767,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,511,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,32767,0.05186666548252106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,65535,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,65535,0.10134933392206828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,65535,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,65535,0.08698667089144389
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,131071,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,131071,0.1873706579208374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,131071,0.015642666568358738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,15,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,63,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,127,0.011519999553759893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1023,0.012293333808581034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,2047,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,2047,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,4095,0.017344000438849132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,4095,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,8191,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,8191,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,16383,0.039642666776975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,16383,0.02531733363866806
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,131071,0.1586186687151591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,32767,0.039834665755430855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,65535,0.10069333513577779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,65535,0.06001066664854685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,3,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,15,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,63,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,511,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1023,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,2047,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,4095,0.02123733361562093
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,4095,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,8191,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,8191,0.02497600018978119
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,32767,0.06083733340104421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,16383,0.06036800146102905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,16383,0.03978666663169861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,32767,0.10026133060455322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,32767,0.058373332023620605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,2047,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,3,0.011626667032639185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,65535,0.18263999621073404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,65535,0.09921600421269734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,15,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,255,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,511,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,511,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1023,0.014746667196353277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1023,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,2047,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,2047,0.019130667050679524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,4095,0.03977599988381068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,4095,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,8191,0.05875733494758606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,8191,0.039317332208156586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,16383,0.10002666711807251
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,16383,0.05791999896367391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,32767,0.18094400564829508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,32767,0.09723732868830363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,65535,0.34337600072224933
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,65535,0.1784213383992513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,31,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,255,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,255,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,511,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,511,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,2047,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,2047,0.011952000359694162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,4095,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,4095,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,8191,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,8191,0.01600533351302147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,16383,0.019904000063737232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,16383,0.019445333629846573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,32767,0.02739199995994568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,32767,0.02737066646416982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,65535,0.031727999448776245
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,65535,0.030618667602539062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,131071,0.035536001125971474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,131071,0.033930666744709015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,127,0.010373333469033241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,15,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1023,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,2047,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,4095,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,4095,0.012746666868527731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,8191,0.01616000011563301
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,8191,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,16383,0.020453333854675293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,16383,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,32767,0.023333333432674408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,65535,0.02632533262173335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,65535,0.025216000775496166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,32767,0.02457600086927414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,131071,0.029498666524887085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,31,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,63,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,127,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,255,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,2047,0.01268799975514412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,2047,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,4095,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,127,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,4095,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,8191,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,131071,0.02794666588306427
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,16383,0.018853332847356796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,32767,0.021327999730904896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,32767,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,65535,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,65535,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,16383,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,3,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,131071,0.043765331308046974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,131071,0.027802666028340656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,8191,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,127,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,255,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,7,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1023,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,2047,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,2047,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,4095,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,4095,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,8191,0.016757333030303318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,8191,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,16383,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,16383,0.021162666380405426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,511,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,32767,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,65535,0.025370667378107708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,32767,0.02317333221435547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,131071,0.028207999964555103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,3,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,131071,0.03151999910672506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,63,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,65535,0.025594666600227356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,511,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,2047,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,4095,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,4095,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,8191,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,8191,0.01747200017174085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,16383,0.020586666961510975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,16383,0.0189280000825723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,32767,0.021456000705560047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,65535,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,65535,0.02277333289384842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,3,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,131071,0.04531733194986979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,131071,0.0276053324341774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,15,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,32767,0.02314666658639908
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,255,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,7,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,2047,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,2047,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,4095,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,4095,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,8191,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,511,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,16383,0.019920000185569126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,31,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,16383,0.01883200059334437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,32767,0.023002666731675465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,32767,0.01958400011062622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,65535,0.039813332259655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,65535,0.02508266766866048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,3,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,131071,0.059749335050582886
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,8191,0.01711999997496605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,31,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,63,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,131071,0.041690667470296226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1023,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,2047,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,4095,0.021456000705560047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,4095,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,511,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,8191,0.04095466683308283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,8191,0.025018667181332905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,16383,0.060645331939061485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,16383,0.0395413339138031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,32767,0.1018986701965332
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,63,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,255,0.012479999413092932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,255,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,511,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,511,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1023,0.015098666151364645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,2047,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,32767,0.06035199761390686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,2047,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,511,0.012650666137536367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,4095,0.03979199876387914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,8191,0.06030400097370148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,8191,0.03956799954175949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,16383,0.10126933455467224
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,16383,0.05798399945100149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,4095,0.023381332556406658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,32767,0.18157867590586343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,32767,0.09754133224487305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,7,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,7,0.012645332763592402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,15,0.012703999876976013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,15,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,31,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,31,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,3,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,63,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,127,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,127,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,3,0.013568000247081121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,255,0.01221866657336553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,511,0.01599466676513354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,511,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1023,0.019424000134070713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,2047,0.040565334260463715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,2047,0.02436800052722295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,63,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,4095,0.05949333310127258
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,4095,0.0400693342089653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,255,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,8191,0.10158933202425639
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1023,0.016544000556071598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,16383,0.18289067347844443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,16383,0.10110933581988017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,32767,0.18269334236780801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,8191,0.0606879989306132
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,7,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,127,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,255,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,511,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,511,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1023,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1023,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,2047,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,2047,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,4095,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,4095,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,8191,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,8191,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,16383,0.021013334393501282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,16383,0.021407999098300934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,32767,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,32767,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,32767,0.3482079903284709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,7,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,65535,0.025514667232831318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,65535,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,7,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,131071,0.029418667157491047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,511,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1023,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,2047,0.014085333794355392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,2047,0.01423466702302297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,4095,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,511,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,4095,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,8191,0.019487999379634857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,8191,0.017397332936525345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,3,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,16383,0.02146133283774058
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,16383,0.01912533367673556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,32767,0.023599999646345775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,32767,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,65535,0.04238399863243103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,65535,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,131071,0.04610133171081543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,131071,0.04177066683769226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,7,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,15,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,131071,0.06187733511130015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,255,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,511,0.011717333147923151
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,2047,0.014901333798964819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,2047,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,4095,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,4095,0.01684800038735072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,8191,0.0195573332409064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,8191,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,511,0.012266666938861212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,16383,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,32767,0.04030933231115341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,32767,0.025098666548728943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,65535,0.058880001306533813
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,16383,0.023226665953795116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,3,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,131071,0.0990826686223348
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,65535,0.03980266551176707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,31,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,255,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1023,0.014698666830857595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1023,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,2047,0.021349333226680756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,2047,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,4095,0.04141333450873693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,4095,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,8191,0.05972800155480703
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,8191,0.039706667264302574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,131071,0.05995733539263407
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,16383,0.10129599769910176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,16383,0.05898666878541311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,7,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,7,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,15,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,15,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,31,0.014495999862750372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,63,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,3,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,63,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,127,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,127,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,255,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,511,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,511,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1023,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1023,0.016789333273967106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,2047,0.040762667854626976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,2047,0.02532266577084859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,3,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,4095,0.061280002196629844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,4095,0.03997333347797394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,8191,0.10131200154622395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,8191,0.06107200185457865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,31,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,16383,0.1016533374786377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,3,0.015370666980743408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,3,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,7,0.015311999867359797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,7,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,16383,0.1846026579538981
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,15,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,15,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1,0.014314666390419006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,31,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,127,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,127,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,255,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,255,0.014501333236694336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,511,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,511,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1023,0.03541333228349686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,31,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1023,0.02272533377011617
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,2047,0.06048533320426941
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,2047,0.04137066751718521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,4095,0.1016426682472229
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,4095,0.06172800064086914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,8191,0.18524799744288126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,8191,0.1030506690343221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,16383,0.35100801785786945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,16383,0.18503467241923013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,3,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,3,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,7,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,15,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,15,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,31,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,31,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,63,0.013461332768201828
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,7,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,127,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,255,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,511,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,511,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1023,0.018981333822011948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1023,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,2047,0.03950933367013931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,2047,0.025498665869235992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,4095,0.06128533184528351
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,4095,0.03991466760635376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,3,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,3,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,7,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,15,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,15,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,31,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,127,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,63,0.015285332997639975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,127,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,255,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,255,0.015322666615247726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,511,0.01932799940307935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,511,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1023,0.035445332527160645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1023,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,31,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,2047,0.06060799956321716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,63,0.014794666320085526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1,0.020074666788180668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,4095,0.06241066753864288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,3,0.019274666905403137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,3,0.01869333287080129
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,7,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,7,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,15,0.01941866676012675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,15,0.01777600000301997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,31,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,2047,0.03973866750796636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,63,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,4095,0.10359467069307964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,127,0.019194666296243668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,127,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,255,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,255,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,511,0.03262399882078171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,511,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1023,0.0531626691420873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1023,0.03526400029659271
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,2047,0.09550399581591289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,2047,0.05834133426348368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,4095,0.17096533377965292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,4095,0.09437333544095357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,63,0.017349333812793095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,3,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,3,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,7,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,7,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,15,0.01544533297419548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,15,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,31,0.015237333873907724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,31,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,63,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,63,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,127,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,127,0.015599999576807022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,255,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,255,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,511,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,511,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1023,0.03551999976237615
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1023,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,2047,0.061797335743904114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,2047,0.04029866556326548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,3,0.021061333517233532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,3,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,7,0.02067733307679494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,7,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,15,0.0207893339296182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,15,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1,0.019167999426523846
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,31,0.01722666621208191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,63,0.020997333029905956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,31,0.017653333644072216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,63,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,127,0.021381333470344543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,127,0.017429333180189133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,255,0.019215999792019527
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,511,0.0337119996547699
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,31,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,511,0.023391999304294586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1023,0.054485330979029335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1023,0.03544000039498011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,2047,0.09777067104975383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1,0.02938666691382726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,2047,0.05940799911816915
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1,0.025578667720158894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,3,0.02536533276240031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,7,0.029301332930723827
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,7,0.025221332907676697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,15,0.029904000461101532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,15,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,31,0.029530666768550873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,255,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,63,0.02958933264017105
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,3,0.02958400050799052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,63,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,127,0.029338667790095013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,127,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,255,0.03349866718053818
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,255,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,511,0.05282666782538096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,511,0.037402667105197906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1023,0.09309333562850952
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1023,0.05453866720199585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,31,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,127,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1023,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,2047,0.17246399323145548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,2047,0.013786666095256805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,4095,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,4095,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,8191,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,8191,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1023,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,16383,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,16383,0.019381333142518997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,32767,0.023157333334287006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,32767,0.021066665649414062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,65535,0.0403466671705246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,65535,0.025637333591779072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,2047,0.09741866588592529
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,131071,0.062277331948280334
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,3,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,131071,0.041936000188191734
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,511,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1023,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,2047,0.01738133281469345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,4095,0.01743999992807706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,4095,0.01730666682124138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,8191,0.019578666736682255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,8191,0.018698666244745255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,16383,0.02481066683928172
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,31,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,32767,0.039621333281199135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,32767,0.025301332275072735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,65535,0.05932799975077311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,65535,0.03990400085846583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,16383,0.021418665846188862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,2047,0.014848000059525171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,7,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,131071,0.0603359987338384
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,511,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1023,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1023,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,2047,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,2047,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,4095,0.017194667210181553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,131071,0.09902933239936829
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,4095,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,8191,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,16383,0.038005332152048744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,16383,0.023221333821614582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,32767,0.05834133426348368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,32767,0.03823466598987579
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,65535,0.09961600104967754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,65535,0.058378666639328
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,8191,0.021253332495689392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1,0.02125866711139679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,131071,0.18341867129007974
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,3,0.021589333812395733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,3,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,7,0.02117866774400075
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,7,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,15,0.02162666618824005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,15,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,131071,0.10016000270843506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,31,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,63,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,63,0.01940800001223882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,127,0.02088533341884613
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,127,0.018992000569899876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,255,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,255,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,511,0.034330666065216064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,31,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,511,0.02351466566324234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1023,0.053898667295773826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1,0.02935466667016347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1023,0.03554133325815201
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1,0.025727999707063038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,3,0.029525332152843475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,7,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,7,0.025472000241279602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,15,0.029440000653266907
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,15,0.027258666853109997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,31,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,31,0.025674665967623394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,63,0.02924266705910365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,3,0.027104000250498455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,63,0.025605333348115284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,127,0.029418667157491047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,127,0.027061333258946735
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,255,0.033626665671666466
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,255,0.025216000775496166
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,511,0.05415999889373779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,511,0.03774933268626531
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1023,0.09289066990216573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1,0.04773866633574168
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1023,0.05589333176612854
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1,0.04154133299986521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,3,0.047781333327293396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,3,0.04145599901676178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,7,0.04776533444722494
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,7,0.04162666698296865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,15,0.041690667470296226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,15,0.04857600231965383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,31,0.04186666508515676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,63,0.047872001926104225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,63,0.04188266893227895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,127,0.05012799799442291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,127,0.04088533421357473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,31,0.048154667019844055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,511,0.09233066439628601
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,511,0.060122668743133545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,255,0.05427733560403188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1023,0.09522133072217305
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,1,0.03201066702604294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,1,0.029365333418051403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,255,0.04287999868392944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,3,0.03164266546567281
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,3,0.028837333122889202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,7,0.03140799949566523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,7,0.029205332199732464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,15,0.03187733391920725
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1023,0.1681706706682841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,15,0.02934933453798294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,31,0.03172266731659571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,63,0.031504000226656594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,63,0.028005334238211315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,127,0.03294399877389272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,127,0.02769600103298823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,31,0.02755733331044515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,255,0.02757333219051361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,511,0.05795733133951823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,511,0.039834665755430855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,1,0.05037866532802582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,1,0.041797334949175514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,3,0.04980266590913137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,3,0.04256533086299896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,7,0.05008000135421753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,255,0.03619199991226196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,7,0.041797334949175514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,15,0.05021866659323374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,15,0.04185600082079569
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,31,0.049914668003718056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,63,0.050106664498647056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,63,0.04182933270931244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,127,0.05231999854246775
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,127,0.04191466669241587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,31,0.04201599955558777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,255,0.056405335664749146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,255,0.045408000548680626
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,511,0.0953439970811208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,511,0.06217599908510844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,1,0.07239999870459239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,3,0.08889066179593404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,3,0.0726506660381953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,7,0.08878399928410848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,1,0.08894399801890056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,7,0.07286400099595387
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,15,0.0886240005493164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,15,0.07293866574764252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,31,0.08889599641164143
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,31,0.0732586681842804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,63,0.08899733424186707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,63,0.07258133093516032
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,127,0.09028266867001851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,127,0.0745600014925003
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,255,0.09595200419425964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,511,0.17068266868591309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,31,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,255,0.07582933207352956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,511,0.11269332965215047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,511,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1023,0.012015999605258306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1023,0.012469333906968435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,2047,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,4095,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,8191,0.01950399950146675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,8191,0.018677332748969395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,16383,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,2047,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,16383,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,4095,0.016879999389251072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,32767,0.041290665666262306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,32767,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,65535,0.0605973352988561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,65535,0.04018666595220566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,3,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,7,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,31,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,131071,0.06144533554712931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1023,0.01267733300725619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1023,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,131071,0.10042666395505269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,2047,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,4095,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,8191,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,8191,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,16383,0.03967999915281931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,16383,0.023311999936898548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,32767,0.06042666733264923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,32767,0.03807466725508372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,4095,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,511,0.01379199946920077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,65535,0.058229332168896995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,7,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,131071,0.1844373345375061
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,7,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,131071,0.1013706624507904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,255,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,511,0.011744000017642975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1023,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,2047,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,2047,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,4095,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,4095,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,8191,0.03937066594759623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,8191,0.023647998770078022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,16383,0.03938666731119156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,32767,0.10014399886131287
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,32767,0.05841066439946493
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,65535,0.10134399930636089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,65535,0.18156800667444864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,65535,0.09780800342559814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1,0.011823999385039011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,3,0.011605333536863327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,3,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,16383,0.05871999760468801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,7,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,131071,0.34745065371195477
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,131071,0.17941866318384805
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,7,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,31,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,31,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,63,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,63,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,127,0.010405333091815313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,127,0.01184533288081487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,127,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,511,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,511,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1023,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1023,0.011621333658695221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1023,0.012709333250919977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,2047,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,2047,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,2047,0.01533866673707962
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,4095,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,4095,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,4095,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,4095,0.01725333308180173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,8191,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,8191,0.018746666610240936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,8191,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,8191,0.01932266727089882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,16383,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,16383,0.023269332945346832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,16383,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,16383,0.022602667411168415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,32767,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,32767,0.03972800076007843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,31,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,32767,0.031311998764673867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,65535,0.013424000392357508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,65535,0.06029333174228668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,65535,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,65535,0.052245333790779114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,131071,0.015413332730531693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,131071,0.10432533423105876
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,131071,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,131071,0.08899733424186707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,3,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,3,0.011530666301647821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,7,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,31,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,63,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,127,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,127,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,255,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,511,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,511,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1023,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1023,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1023,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,2047,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,2047,0.016949333250522614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,2047,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,2047,0.01676799977819125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,4095,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,4095,0.018800000349680584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,4095,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,63,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,4095,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,8191,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,8191,0.02342933416366577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,8191,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,8191,0.023178666830062866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,16383,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,16383,0.039674667020638786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,16383,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,16383,0.03148799886306127
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,32767,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,32767,0.06042666733264923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,32767,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,32767,0.05217066903909048
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,65535,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,65535,0.10172266761461894
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,65535,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,65535,0.08685866991678874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,131071,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,131071,0.18758400281270346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,131071,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,3,0.010543999572594961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,7,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,7,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,7,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,15,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,15,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,63,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,255,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,255,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1023,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1023,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,2047,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,2047,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,2047,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,2047,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,4095,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,4095,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,4095,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,4095,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,131071,0.1585706671079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,8191,0.014805333067973455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,8191,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,8191,0.015210667004187902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,16383,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,16383,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,16383,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,16383,0.017674667139848072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,32767,0.02351466566324234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,32767,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,8191,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,65535,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,65535,0.029717333614826202
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,32767,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,65535,0.028255999088287354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,32767,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,131071,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,131071,0.032032000521818794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1,0.010464000205198923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,131071,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1,0.012128000458081564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,65535,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,31,0.01251199965675672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,63,0.011493333925803503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,127,0.010586666564146677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,127,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,255,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1023,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1023,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,2047,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,255,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1,0.010522666076819101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,2047,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,4095,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,131071,0.03345066557327906
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,4095,0.011781333635250727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,4095,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,4095,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,8191,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,8191,0.015557333827018738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,8191,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,8191,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,16383,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,2047,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,2047,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,16383,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,32767,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,32767,0.020373333245515823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,32767,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,32767,0.020058666666348774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,65535,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,16383,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,16383,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,65535,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,65535,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,131071,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,65535,0.021130666136741638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,131071,0.02752000093460083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,131071,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,3,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,131071,0.02609066665172577
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,7,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,15,0.010602666685978571
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,31,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,63,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,127,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,255,0.010501333822806677
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,255,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,511,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,2047,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,2047,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,2047,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,2047,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,4095,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,4095,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,4095,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,8191,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,8191,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,8191,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,8191,0.016016000260909397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,4095,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,16383,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,16383,0.019296000401178997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,16383,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,16383,0.019413333386182785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,32767,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,32767,0.02257599929968516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,32767,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,32767,0.023157333334287006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,65535,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,65535,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,65535,0.0230880007147789
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,131071,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,65535,0.023631999890009563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,131071,0.02775466690460841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,131071,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1,0.01055466632048289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,3,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,15,0.0100426667680343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,15,0.0099093330403169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,131071,0.027082666754722595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,63,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,63,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,127,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,127,0.010319999729593595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,511,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,15,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,511,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1023,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1023,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,2047,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,2047,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,2047,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,511,0.011541333049535751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,2047,0.012970666090647379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,4095,0.012250666817029318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,4095,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,4095,0.014362666755914688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,8191,0.011909333368142446
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,8191,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,8191,0.01685333376129468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,16383,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,16383,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,16383,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,16383,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,32767,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,32767,0.02035733312368393
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,32767,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,32767,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,65535,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,65535,0.021354667842388153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,65535,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,65535,0.021557333568731945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,8191,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,131071,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,131071,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,131071,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,3,0.010640000303586325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,31,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,131071,0.027503999571005504
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,63,0.011637333780527115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,127,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,127,0.011813333878914515
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,127,0.011653333902359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,255,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,255,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,511,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,511,0.011861333002646765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,511,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,511,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1023,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1023,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1023,0.012805332740147909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,2047,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,2047,0.017621333400408428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,2047,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,2047,0.015850666910409927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,4095,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,4095,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,4095,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,8191,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,8191,0.025221332907676697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,8191,0.01328533391157786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,8191,0.023200000325838726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,16383,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,16383,0.03956266740957896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,16383,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,16383,0.031328000128269196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,32767,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,32767,0.06102400024731954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,32767,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,65535,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,32767,0.05240533252557119
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,65535,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,3,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,65535,0.1027946670850118
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,3,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,65535,0.08679999907811482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,7,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,31,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,511,0.011839999506870905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,511,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,511,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1023,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,3,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1023,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,2047,0.014730667074521383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,2047,0.019023999571800232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,2047,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,2047,0.017674667139848072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1023,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,4095,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,4095,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,4095,0.021173333128293354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,8191,0.01301866645614306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,8191,0.03827200084924698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,8191,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,8191,0.031130666534105938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,4095,0.013343999783198038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,16383,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,16383,0.058789332707722984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,16383,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,16383,0.05215999980767568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,32767,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,32767,0.10131733616193135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,32767,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,32767,0.08705066641171773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,65535,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,65535,0.18402665853500366
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,65535,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,65535,0.15802666544914246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,7,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,7,0.01027199998497963
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,7,0.010480000327030817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,31,0.010533332824707031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,63,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,255,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,255,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,127,0.010549332946538925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1023,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1023,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,2047,0.01259200026591619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,2047,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,4095,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,511,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,4095,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,4095,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,4095,0.01444799949725469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,8191,0.012703999876976013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,8191,0.01756799966096878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,8191,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,8191,0.01883200059334437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,16383,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,16383,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,2047,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,16383,0.01915733392039935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,2047,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,32767,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,32767,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,32767,0.020848001043001812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,65535,0.013258667041858038
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,65535,0.023168000082174938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,16383,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,65535,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,32767,0.021301334102948506
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,65535,0.023183998962243397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,131071,0.013317332913478216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,131071,0.029872000217437744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1,0.011674666156371435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,3,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,7,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,131071,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,131071,0.027535999814669292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,15,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,63,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,15,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,3,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,255,0.010591999938090643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,511,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,511,0.011402666568756104
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1023,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1023,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,2047,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,2047,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,2047,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,4095,0.012714666624863943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,4095,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,4095,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,4095,0.016224000602960587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,8191,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,8191,0.01766933376590411
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,8191,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,8191,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,16383,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,16383,0.019061333189407986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1023,0.011887999872366587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,16383,0.018016000588734944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,32767,0.013167999684810638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,32767,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,32767,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,65535,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,16383,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,65535,0.025114665428797405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,65535,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,32767,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,65535,0.023898666103680927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,131071,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,131071,0.04177600145339966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,131071,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,3,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,3,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,3,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,7,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,127,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,15,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,511,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,511,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1023,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,2047,0.01332266628742218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,2047,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,2047,0.013888000200192133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,131071,0.033301333586374916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,2047,0.017653333644072216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,4095,0.01524266724785169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,4095,0.015226667126019796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,4095,0.023002666731675465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,8191,0.014757333944241205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,4095,0.02362666775782903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,8191,0.03966933240493139
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,8191,0.013978666315476099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,255,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,8191,0.03195200115442276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,16383,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,16383,0.05952533086140951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,16383,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,16383,0.05237866441408793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,32767,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,32767,0.1013813316822052
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,3,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,32767,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,7,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,7,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,15,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,15,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,15,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,31,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,31,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,63,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,63,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,63,0.012117333710193634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,63,0.011642667154471079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,32767,0.08848533034324646
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,127,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,255,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,255,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,255,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,255,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,511,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,511,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,511,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,511,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1023,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1023,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1023,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,2047,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,2047,0.023445333043734234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,2047,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,2047,0.023007998863856
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,4095,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,4095,0.03972266614437103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,4095,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,4095,0.03163733333349228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,7,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,8191,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,8191,0.06155733267466227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,8191,0.015461333096027374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,8191,0.05359466870625814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,16383,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,16383,0.10354133447011311
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1023,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,16383,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,16383,0.08905067046483357
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,32767,0.015439999600251516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,32767,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,3,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,3,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,32767,0.18645866711934408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,3,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,7,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,32767,0.15874133507410684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,7,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,7,0.012250666817029318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,31,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,63,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,63,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,63,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,127,0.012175999581813812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,127,0.012602667013804117
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,127,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,127,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,255,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,7,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,255,0.014576000471909841
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,255,0.01339200014869372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,255,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,511,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,511,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1023,0.015840000162522
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1023,0.01522133375207583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,2047,0.015333333363135656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,2047,0.023402666052182514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,2047,0.02309866746266683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,4095,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,511,0.013658666362365087
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,4095,0.03942399968703588
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,4095,0.015279999623696009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,4095,0.0315733328461647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,2047,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,8191,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,8191,0.06122133135795593
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,8191,0.05406933526198069
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,3,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,3,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,3,0.015018666783968607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,7,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,7,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,7,0.013162666310866674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,7,0.014885333677132925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,15,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,15,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,15,0.013104000439246496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,15,0.013130666067202887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,31,0.01341333364446958
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,31,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,31,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,31,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,63,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,8191,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1,0.014607999473810196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,127,0.014661333213249842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,127,0.013376000026861826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,127,0.014250667144854864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,127,0.014725333700577417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,255,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,255,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,63,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,255,0.01509333277742068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,511,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,511,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,511,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,511,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1023,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1023,0.02309333284695943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1023,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1023,0.021189334491888683
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,63,0.01322666679819425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,2047,0.04021333406368891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,2047,0.03142400085926056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,4095,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,4095,0.06253333389759064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,255,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,2047,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,4095,0.05427733560403188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,8191,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,8191,0.1053546667098999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,8191,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,4095,0.01739199956258138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1,0.01482133318980535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,3,0.015013333410024643
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,3,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,3,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,7,0.014815999815861383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,7,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,7,0.014874666929244995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,3,0.014426667243242264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,7,0.013301332791646322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,15,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,15,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,15,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,31,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,31,0.013440000514189402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,31,0.013221333424250284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,31,0.013653332988421122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,63,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,63,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,63,0.014997333288192749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,63,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,127,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,127,0.015200000256299973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,127,0.014853333433469137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,255,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,127,0.014010666559139887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,8191,0.0886346697807312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,255,0.013157332936922709
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,511,0.014538666854302088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,511,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,511,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,511,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1023,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,15,0.014159999787807465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1023,0.02316266546646754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1023,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,255,0.013440000514189402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,2047,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,2047,0.039642666776975
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,2047,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,2047,0.03329599897066752
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,4095,0.017573333034912746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,4095,0.061648001273473106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,4095,0.017504000415404636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1023,0.021274665991465252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,3,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,3,0.01798933371901512
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,3,0.016976000120242436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,3,0.017594666530688603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,7,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,7,0.018079999834299088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,7,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,7,0.017184000462293625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,15,0.017946666727463405
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,15,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,15,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,15,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,31,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,31,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,31,0.01747200017174085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,31,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,63,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,4095,0.054383998115857445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,63,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,63,0.017504000415404636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,127,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,127,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,127,0.017498667041460674
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,127,0.01692266638080279
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,255,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,255,0.017397332936525345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,255,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,255,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,511,0.01882133384545644
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,63,0.017893332988023758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,511,0.022778667509555817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,511,0.017509333789348602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,511,0.021104000508785248
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1023,0.017935999979575474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1023,0.033733333150545754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1023,0.01714133347074191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1023,0.029605334003766377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,2047,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,2047,0.05809600154558817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,2047,0.05108266572157542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,4095,0.020960000654061634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,4095,0.09630399942398071
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,4095,0.020010666300853092
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,4095,0.08152000109354655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,3,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,3,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,7,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,31,0.011503999431928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,63,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,63,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,511,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,2047,0.021130666136741638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1023,0.01145600030819575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,2047,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,2047,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,2047,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,4095,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,4095,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,4095,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,8191,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,8191,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,8191,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,8191,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,16383,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,16383,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,16383,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,16383,0.018917333334684372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,2047,0.013823999712864557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,32767,0.021520001192887623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1023,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,32767,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,32767,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,32767,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,65535,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,65535,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,65535,0.023397333920001984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,65535,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,131071,0.014890667051076889
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,131071,0.04204266766707102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,131071,0.014789332946141561
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,15,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,63,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,127,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,131071,0.03352533280849457
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,31,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,511,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1023,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,255,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,2047,0.013349333157142004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,2047,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,2047,0.01505600040157636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,4095,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,4095,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,4095,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,8191,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,8191,0.01773333301146825
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,8191,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,8191,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,511,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,16383,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,16383,0.02126399924357732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,16383,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,32767,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,32767,0.02502399931351344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,16383,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,32767,0.025258667767047882
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,4095,0.01682666689157486
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,32767,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,65535,0.03990933299064636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,65535,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,65535,0.031146667897701263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,131071,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,131071,0.060565332571665444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,131071,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,65535,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1,0.017525333911180496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,131071,0.054229333996772766
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,3,0.017509333789348602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,3,0.017258666455745697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,3,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,7,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,7,0.017525333911180496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,7,0.01703466723362605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,7,0.017423999806245167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,3,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,15,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,15,0.017743999759356182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,15,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,15,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1,0.017301333447297413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,31,0.017637333522240322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,31,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,31,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,31,0.017322666943073273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,63,0.01756799966096878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,63,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,63,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,127,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,127,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,127,0.0174346665541331
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,127,0.01710933322707812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,255,0.01736533393462499
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,63,0.017263999829689663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,255,0.017701332767804463
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,255,0.017514667163292568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,255,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,511,0.0191040001809597
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,511,0.022991999983787537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,511,0.017103999853134155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,511,0.021151999632517498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1023,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1023,0.035455999275048576
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1023,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1023,0.02939733366171519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,2047,0.02139200021823247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,2047,0.058543999989827476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,2047,0.02075200031201045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,2047,0.04994666576385498
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1,0.025402667621771496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1,0.02553066611289978
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1,0.024570666253566742
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1,0.023226665953795116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,3,0.025263999899228413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,3,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,3,0.02535466601451238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,3,0.023189333577950794
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,7,0.02526933451493581
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,7,0.02534399926662445
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,7,0.025120000044504803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,7,0.023205332458019257
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,15,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,15,0.025797332326571148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,15,0.025120000044504803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,15,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,31,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,31,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,31,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,31,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,63,0.02510400116443634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,63,0.025418666501839954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,63,0.02502399931351344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,127,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,127,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,127,0.02477866659561793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,127,0.023386667172114056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,255,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,255,0.025472000241279602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,255,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,63,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,511,0.02550933261712392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,511,0.03757333258787791
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,511,0.025370667378107708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,511,0.03178133318821589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1023,0.025733334322770435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1023,0.05625600119431814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1023,0.02456533412138621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1023,0.04938133557637533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,2047,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,2047,0.09795199831326802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,2047,0.027322667340437572
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,2047,0.08457066615422566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1,0.023178666830062866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,255,0.023455999791622162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1,0.023573334018389385
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,3,0.025605333348115284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,3,0.02550400048494339
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,3,0.02566933383544286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,3,0.023418667415777843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,7,0.025237334271272022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,7,0.025519999365011852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,7,0.023376000424226124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,15,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,15,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,15,0.0233599990606308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,15,0.023658665517965954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,7,0.02514133354028066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,31,0.025386666258176167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,31,0.025333332518736523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,31,0.023706667125225067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,31,0.025045332809289295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,63,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,63,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,63,0.025263999899228413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,63,0.025514667232831318
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,127,0.02548266698916753
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,127,0.025205334027608235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,127,0.024832000335057575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,127,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,255,0.02718399961789449
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,255,0.025621332228183746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,255,0.025242666403452556
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,511,0.026591998835404713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,511,0.037871999045213066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,511,0.025397333006064098
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,511,0.03183999905983607
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1023,0.02677333354949951
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1023,0.025642665723959606
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1023,0.04996266464392344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,255,0.02366400013367335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1,0.03997333347797394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1,0.03987200061480204
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1,0.037503999968369804
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,3,0.039962666730086006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1023,0.056234667698542275
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,3,0.041802664597829185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,3,0.03750933210055033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1,0.03772266705830892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,3,0.037461332976818085
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,7,0.04146666576464971
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,7,0.03990933299064636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,7,0.03754133234421412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,7,0.03751999884843826
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,15,0.04009066770474116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,15,0.03749333322048187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,15,0.03793599953254064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,31,0.0397173340121905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,31,0.04148799926042557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,31,0.03765333443880081
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,31,0.037861332297325134
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,63,0.0415786678592364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,63,0.039919999738534294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,63,0.03786666691303253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,15,0.042223999897638954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,127,0.039594667653242745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,127,0.03969600051641464
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,127,0.038015998899936676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,63,0.038015998899936676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,255,0.04238399863243103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,255,0.04199466605981191
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,255,0.039477333426475525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,255,0.03746666759252548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,511,0.04155733436346054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,511,0.062090665102005005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,511,0.03827200084924698
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,511,0.05427733560403188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1023,0.04318933188915253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,127,0.03754133234421412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1023,0.03762666632731756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,15,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,15,0.010661333799362183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,31,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1023,0.08478933572769165
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1023,0.09795733292897542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,127,0.010565333068370819
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,255,0.010506667196750641
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,255,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,511,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1023,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1023,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1023,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,2047,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,4095,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,4095,0.01724799970785777
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,4095,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,511,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,8191,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,8191,0.019205333044131596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,8191,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,8191,0.01749333366751671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,16383,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,16383,0.020975999534130096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,16383,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,2047,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,16383,0.01929066702723503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,4095,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,32767,0.025061334172884624
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,32767,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,32767,0.023792001108328503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,65535,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,65535,0.04009599983692169
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,65535,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,65535,0.0315733328461647
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,131071,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,131071,0.06020799775918325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,131071,0.053130666414896645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,7,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,131071,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,7,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,15,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,31,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,31,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,127,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,127,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,511,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,511,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1023,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1023,0.012768000364303589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,2047,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,2047,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,2047,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,4095,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,4095,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,4095,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1,0.01126933346192042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,8191,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,8191,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,8191,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,8191,0.017210666090250015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,16383,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,4095,0.015541333705186844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,15,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,16383,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,16383,0.02179733415444692
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,32767,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,32767,0.03955733279387156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,32767,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,16383,0.023728000621000927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,32767,0.031632001201311745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,65535,0.013232000172138214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,65535,0.060362666845321655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,65535,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,65535,0.05218133330345154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,131071,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,131071,0.1048959990342458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,131071,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,131071,0.08874666690826416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,3,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,7,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,15,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,127,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,255,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,511,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,511,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1023,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,2047,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,2047,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,4095,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,8191,0.0215786670645078
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,8191,0.019093333433071773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,16383,0.03956266740957896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,16383,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,32767,0.06036800146102905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,32767,0.039818666875362396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,255,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,65535,0.10224533081054688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,65535,0.058933332562446594
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,3,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,7,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,7,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,15,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,31,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,63,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,131071,0.1848426659901937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,127,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,255,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,511,0.011855999628702799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,511,0.011957333733638128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1023,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1023,0.013274667163689932
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,2047,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,2047,0.01714666684468587
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,4095,0.019039999693632126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,8191,0.04012266546487808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,8191,0.023237332701683044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,16383,0.05975466469923655
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,4095,0.021727999051411945
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,32767,0.10123200217882793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,32767,0.05959466596444448
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,131071,0.10140267014503479
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,16383,0.03953066716591517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,65535,0.09843200445175171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1,0.010559999694426855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,15,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,31,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,31,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,131071,0.1803306738535563
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,63,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,65535,0.18257067600886026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,2047,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,2047,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,4095,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,4095,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,8191,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,131071,0.3476053476333618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,16383,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,16383,0.021183999876181286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,32767,0.027471999327341717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,32767,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,65535,0.030053332448005676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,8191,0.015418666104475657
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,131071,0.03484266748030981
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,31,0.010650667051474253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,31,0.010543999572594961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,255,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,511,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,2047,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,4095,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,4095,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,131071,0.03367999941110611
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,8191,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,16383,0.02107200026512146
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,16383,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,65535,0.03048533449570338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,32767,0.02313599983851115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,32767,0.023226665953795116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,65535,0.025562666356563568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,65535,0.02329600105683009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,8191,0.015439999600251516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,3,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,131071,0.02926933268706004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,63,0.011424000064531961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,255,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,511,0.010794666906197866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1023,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,2047,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,2047,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,4095,0.012634667257467905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,8191,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,8191,0.016890666137139004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,16383,0.02070933332045873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,16383,0.020831999679406483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,32767,0.023141334454218548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,32767,0.023237332701683044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,65535,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,65535,0.025583999852339428
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,3,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,131071,0.031680000325044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,131071,0.02737066646416982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1023,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,2047,0.012826666235923767
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,2047,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,4095,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,4095,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,8191,0.01828266680240631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,8191,0.016943999876578648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,16383,0.021130666136741638
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,16383,0.01905599981546402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,32767,0.021525333325068157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,65535,0.02555199960867564
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,131071,0.027535999814669292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,65535,0.023365333676338196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,32767,0.02109866589307785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,7,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,31,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,511,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,511,0.012106666962305704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1023,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,2047,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,131071,0.04514666895071665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,4095,0.021082667013009388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,4095,0.019306667149066925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,8191,0.04307733476161957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,8191,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,16383,0.06058133145173391
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,16383,0.039461334546407066
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,131071,0.027317332724730175
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,32767,0.10175999999046326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,32767,0.05994666616121928
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,3,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,65535,0.1818293333053589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,7,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,65535,0.09916800260543823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,31,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,127,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,511,0.012341332932313284
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1023,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,2047,0.020421333611011505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,2047,0.018725333114465077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,4095,0.0393653338154157
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,4095,0.023002666731675465
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,8191,0.06002666552861532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,8191,0.03932799895604452
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,16383,0.10077866911888123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,16383,0.05819733440876007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,32767,0.18213866154352823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,32767,0.09890133142471313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1,0.010608000059922537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,7,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,7,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,15,0.011589333415031433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,15,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,65535,0.3435200055440267
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,31,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,127,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,3,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,511,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,511,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1023,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,2047,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,2047,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,4095,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,8191,0.018976000448067982
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,8191,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,16383,0.021029333273569744
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,65535,0.17898666858673096
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,16383,0.020661332954963047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,32767,0.023183998962243397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,65535,0.027530667682488758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,65535,0.023418667415777843
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1,0.011653333902359009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,3,0.011855999628702799
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,32767,0.021520001192887623
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,3,0.011584000041087469
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,7,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,127,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,127,0.011434666812419891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,255,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,131071,0.047269334395726524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,131071,0.029557332396507263
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,511,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,2047,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,2047,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,4095,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,4095,0.017338667064905167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,8191,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,8191,0.01911466692884763
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,16383,0.021216000119845074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,16383,0.01933866615096728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,32767,0.023354666928450268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,32767,0.021040000021457672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,65535,0.041840001940727234
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,65535,0.025594666600227356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,131071,0.0620959997177124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,131071,0.043840001026789345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,63,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,3,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1023,0.014666666587193808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,2047,0.020831999679406483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,2047,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,4095,0.04016000032424927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,4095,0.023071999351183575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,8191,0.060677334666252136
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,8191,0.03984533250331879
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,16383,0.1014453371365865
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,16383,0.05925333499908447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1,0.012938667088747025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,511,0.012752000242471695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,3,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,3,0.012634667257467905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,7,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,7,0.01231466606259346
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,15,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,15,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,31,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,31,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,63,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,63,0.012543999900420507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,127,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,127,0.012133333832025528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,255,0.012858666479587555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,255,0.01181866725285848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,511,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,32767,0.18323200941085815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1023,0.01931200052301089
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,32767,0.09943466385205586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1023,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,2047,0.04005866746107737
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,4095,0.06150400141874949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,4095,0.04156800111134847
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,8191,0.1025279959042867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,2047,0.02518933266401291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,511,0.014186666657527288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,16383,0.18396800756454468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,16383,0.1011893351872762
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,8191,0.061039999127388
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,32767,0.3487519820531209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,3,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,32767,0.18293333053588867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,3,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,7,0.013082666943470636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,7,0.012650666137536367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,15,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,15,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,31,0.014837333311637243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,31,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,63,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,63,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,127,0.012789333860079447
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,127,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,255,0.013264000415802002
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,255,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,511,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,511,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1023,0.018826667219400406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1023,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,2047,0.04010133445262909
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,2047,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,4095,0.061792001128196716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,4095,0.03974399964014689
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,8191,0.10364799698193868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,3,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,8191,0.06182399888833364
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,3,0.015194666882356008
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,7,0.015077333897352219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,7,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,15,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,31,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,31,0.014922666052977243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,63,0.015184000134468079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,63,0.015423999478419622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,127,0.015274666249752045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1,0.014202666779359182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,255,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,255,0.013893333574136099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,511,0.019519999623298645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,511,0.01721599946419398
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,15,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1023,0.035445332527160645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1023,0.021354667842388153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,2047,0.06185600161552429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,2047,0.040896000961462654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,4095,0.10243733723958333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,4095,0.06247999767462412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,8191,0.10335466265678406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1,0.015157333264748255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,3,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,127,0.015109332899252573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,7,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,7,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,15,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,8191,0.18593599398930868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,15,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,31,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,63,0.01469333345691363
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,3,0.014991999914248785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,127,0.015130666395028433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,127,0.013898666948080063
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,255,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,511,0.019530666371186573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,511,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1023,0.03533333291610082
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1023,0.024330665667851765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,63,0.015557333827018738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,2047,0.06192533175150553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,2047,0.04223466912905375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1,0.019109333554903667
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,3,0.019280000279347103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,3,0.018874666343132656
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,7,0.019317333896954853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,7,0.018933333456516266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,15,0.019130667050679524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,15,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,31,0.01942933350801468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,31,0.017375999440749485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,4095,0.10388799508412679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,4095,0.06221333146095276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,63,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,127,0.019493332753578823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,127,0.018053332964579265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,255,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,255,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,511,0.033359999457995095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,511,0.02332266668478648
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1023,0.053226664662361145
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1023,0.03369600077470144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,2047,0.09730666875839233
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,2047,0.058149332801500954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,63,0.01926400015751521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,4095,0.17287466923395792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,4095,0.09725333253542583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,63,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,3,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,511,0.011871999750534693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1023,0.011760000139474869
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,2047,0.01562133307258288
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,4095,0.018186666071414948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,4095,0.017594666530688603
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,8191,0.019653332730134327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,8191,0.01930133377512296
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,16383,0.0216799999276797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1023,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,16383,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,2047,0.0138026662170887
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,32767,0.0245919997493426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,32767,0.021359999974568684
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,65535,0.041349334021409355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,511,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,65535,0.025407999753952026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,3,0.011258666714032492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,131071,0.06243733565012614
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,131071,0.043136000633239746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,127,0.011498666057984034
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,255,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,511,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1023,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,2047,0.015578666081031164
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,2047,0.016906666258970898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,4095,0.018960000326236088
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,4095,0.01709866647919019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,8191,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,16383,0.02327466756105423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,16383,0.021365332106749218
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,32767,0.03995733211437861
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,32767,0.025248001019159954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,15,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,8191,0.021381333470344543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,65535,0.05892266829808553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,65535,0.0405173326532046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1,0.01918399954835574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,131071,0.10040533542633057
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,131071,0.060975998640060425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,3,0.021157334248224895
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,7,0.019744000087181728
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,7,0.018944000204404194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,15,0.021125334004561108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,15,0.017114666601022083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,31,0.021210665504137676
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,31,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,63,0.01966933285196622
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,3,0.01907733331123988
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,63,0.018751999984184902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,127,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,127,0.01746133342385292
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,255,0.019226666539907455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,511,0.03339733431736628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,511,0.02317333221435547
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1023,0.05402666827042898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1023,0.03515733281771342
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,255,0.01717866708834966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,2047,0.09773332873980205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1,0.02924266705910365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,2047,0.05852800110975901
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1,0.02537599951028824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,3,0.025733334322770435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,7,0.025781333446502686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,3,0.02942933390537898
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,15,0.02566933383544286
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,15,0.030000001192092896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,7,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,31,0.025477332373460133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,63,0.029338667790095013
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,63,0.025466665625572205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,127,0.029477333029111225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,127,0.025546667476495106
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,255,0.033370666205883026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,255,0.025285333395004272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,31,0.029232000311215717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,511,0.053685332338015236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,511,0.03692800054947535
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1023,0.09296000003814697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1023,0.05592533449331919
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,2047,0.1747093399365743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,2047,0.0976639986038208
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1,0.02993600070476532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1,0.027295999228954315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,3,0.029552000264326733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,3,0.025487999121348064
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,7,0.029504001140594482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,7,0.027589333554108936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,15,0.029232000311215717
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,15,0.025194667279720306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,31,0.030245333909988403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,31,0.027221334477265675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,63,0.029466666281223297
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,63,0.025568000972270966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,127,0.03163733333349228
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,127,0.025797332326571148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,255,0.03373866776625315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,255,0.025648000339667004
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,511,0.0558240016301473
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1,0.04807466765244802
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,511,0.03755199909210205
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1,0.041536000867684685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1023,0.09305066863695781
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,3,0.04187199970086416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,7,0.0481279989083608
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,7,0.04177600145339966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,3,0.04797866443792979
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,15,0.048154667019844055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,15,0.0415040006240209
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,31,0.048138668139775596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,31,0.041749333341916404
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1023,0.05604266623655955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,63,0.0484746644894282
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,63,0.040192000567913055
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,127,0.05054933329423269
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,127,0.039919999738534294
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,255,0.053957333167394005
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,255,0.043562665581703186
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,511,0.09346666932106018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,15,0.011370666325092316
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,511,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1023,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1023,0.16965866088867188
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1023,0.09603200356165568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,2047,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,4095,0.017317333569129307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,8191,0.020992000897725422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,8191,0.01883200059334437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,16383,0.024175999065240223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,2047,0.01545599972208341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,16383,0.02130666623512904
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,32767,0.04166933397452036
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,32767,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,511,0.06072533130645752
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,65535,0.04154666761557261
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,4095,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,65535,0.06154133379459381
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,131071,0.10153067111968994
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,131071,0.061946665247281395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,3,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,3,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,7,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,127,0.010496000448862711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,255,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,31,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1023,0.012074666718641916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,2047,0.015173333386580149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,4095,0.017386666188637417
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,8191,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,16383,0.03921066721280416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,16383,0.02493866781393687
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,32767,0.06046933432420095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,32767,0.03998400022586187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,8191,0.02110933264096578
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,65535,0.05922666688760122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,3,0.010672000547250112
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,3,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,65535,0.09987733761469524
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,131071,0.10120532910029094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,7,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,3,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,127,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1023,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1023,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1023,0.012741333494583765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,131071,0.18452799320220947
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,2047,0.013189333180586496
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,2047,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,511,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,4095,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1023,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,4095,0.015402667224407196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,4095,0.012815999488035837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,4095,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,8191,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,8191,0.019018666197856266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,8191,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,8191,0.01704000060757001
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,2047,0.014933332800865173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,16383,0.023130667706330616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,16383,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,16383,0.021482666333516438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,32767,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,32767,0.03958400090535482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,2047,0.015306666493415833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,16383,0.012831999609867731
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,32767,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,32767,0.030506665507952373
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,65535,0.012842666357755661
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,65535,0.060415998101234436
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,65535,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,65535,0.05217599868774414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,131071,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,131071,0.10506133238474528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,131071,0.015205333630243937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,3,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,3,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,3,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,7,0.011509332805871964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,7,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,31,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,31,0.010666667173306147
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,131071,0.08931733171145122
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,127,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,255,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,255,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,511,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,63,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,511,0.010656000425418219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,511,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1023,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1023,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,2047,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,2047,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,2047,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,2047,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,4095,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,4095,0.012026666353146235
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,4095,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,4095,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,8191,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,8191,0.015135999768972397
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,8191,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,16383,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,16383,0.01922133316596349
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,16383,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,16383,0.019199999670187633
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,32767,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,32767,0.025274666647116344
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,32767,0.011317333827416102
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,32767,0.02610666553179423
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,8191,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,65535,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,65535,0.028832000990708668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,65535,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,65535,0.03145600110292435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,131071,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,131071,0.033743999898433685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,127,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,131071,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1,0.011589333415031433
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,3,0.010373333469033241
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,131071,0.03150933235883713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,31,0.010464000205198923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,63,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,63,0.010399999717871347
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,127,0.010687999427318573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,127,0.010703999549150467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,127,0.010581333190202713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,255,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,511,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,2047,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,2047,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,2047,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,2047,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,4095,0.011391999820868174
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,4095,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,4095,0.011418666690587997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,4095,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,8191,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,8191,0.01695466662446658
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,8191,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,8191,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,16383,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,16383,0.019152000546455383
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,16383,0.011050666371981302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,16383,0.020986666282018025
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,32767,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,32767,0.02292799949645996
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,32767,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,32767,0.023285334308942158
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,65535,0.012613333761692047
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,65535,0.024832000335057575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,65535,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,65535,0.02407466620206833
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,131071,0.012986666212479273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,131071,0.02719466636578242
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,131071,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,3,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,131071,0.025360000630219776
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,15,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,15,0.010821333775917688
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,31,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,63,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,127,0.011365332951148352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,255,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,511,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1023,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1023,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,2047,0.013370666652917862
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,2047,0.015216000378131866
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,2047,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,4095,0.013210666676362356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,4095,0.019248000035683315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,4095,0.01321600005030632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,4095,0.017279999951521557
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,8191,0.013088000317414602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,8191,0.023103999594847362
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,8191,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,8191,0.02145066608985265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,16383,0.013125333935022354
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,16383,0.03963200002908707
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,16383,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,16383,0.031856000423431396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,32767,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,32767,0.06003733476003011
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,32767,0.01421333352724711
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,32767,0.052469333012898765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,65535,0.013050666699806849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,65535,0.10292266805966695
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,65535,0.013173333058754602
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,65535,0.08751466870307922
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,131071,0.015072000523408255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,131071,0.18901334206263223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1,0.011637333780527115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1,0.011445333560307821
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,3,0.01138666644692421
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,3,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,131071,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,7,0.011461333682139715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,7,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,131071,0.15996799866358438
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,15,0.011407999942700068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,15,0.010677333921194077
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,31,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,31,0.01137599969903628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,63,0.010575999816258749
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,127,0.0106133334338665
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,127,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,127,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,255,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,255,0.010480000327030817
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,511,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,511,0.011482667177915573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,511,0.010709332923094431
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1023,0.0116799995303154
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1023,0.011525332927703857
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,2047,0.012784000486135483
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,2047,0.013093333691358566
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,2047,0.013546666751305262
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,4095,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,4095,0.013829333086808523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,15,0.011061333119869232
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,4095,0.014831999937693277
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,8191,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,8191,0.01924266666173935
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,8191,0.01313599944114685
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,8191,0.017711999515692394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,16383,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,16383,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,16383,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,16383,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,32767,0.012965332716703415
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,32767,0.021402666966120403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,4095,0.012975999464591345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,32767,0.021168000996112823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,65535,0.013253333667914072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,65535,0.02327999969323476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,65535,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,65535,0.02332799881696701
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,32767,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,131071,0.0141546664138635
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,131071,0.02993600070476532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1,0.011274666835864386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,131071,0.029578665892283123
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,7,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,131071,0.01492799942692121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,15,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,127,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,511,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,511,0.011877333124478659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,511,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1023,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1023,0.012901333471139273
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,2047,0.018911999960740406
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,2047,0.013269333789745966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,2047,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,4095,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,4095,0.02319466571013133
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,4095,0.023152001202106476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,8191,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,8191,0.03931200007597605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,8191,0.01360000049074491
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,8191,0.0313226655125618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,16383,0.014954666296641031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,4095,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,16383,0.060032000144322716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,16383,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,16383,0.052757332722345986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,32767,0.015066667149464289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,32767,0.10038933157920837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,32767,0.01350933313369751
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,32767,0.08717333277066548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,65535,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,65535,0.01351999988158544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1,0.011546666423479715
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,65535,0.1583573321501414
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1,0.011519999553759893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,3,0.011877333124478659
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,3,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,3,0.012725333372751871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,7,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,7,0.011898666620254517
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,7,0.013072000195582708
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,7,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,15,0.011237333218256632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,15,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,15,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,31,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,31,0.013242666920026144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,31,0.012954667210578918
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,63,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,63,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,127,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,65535,0.18357867002487183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,127,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,255,0.012634667257467905
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,255,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,511,0.012725333372751871
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,511,0.013056000073750814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,511,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1023,0.012837332983811697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1023,0.015829333414634068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1023,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,2047,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,2047,0.023306667804718018
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,2047,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,2047,0.023152001202106476
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,4095,0.015381333728631338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,4095,0.03990400085846583
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,4095,0.015024000157912573
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,4095,0.03155199935038885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,8191,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,8191,0.06201600035031637
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,8191,0.01523200049996376
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,8191,0.05415999889373779
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,16383,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,16383,0.10371200243631999
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,16383,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,16383,0.08933333555857341
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1,0.014938666174809137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1,0.013967999567588171
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,3,0.014837333311637243
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,3,0.014314666390419006
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,3,0.01461333284775416
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,3,0.015114666273196539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,7,0.014202666779359182
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,7,0.014917333920796713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,7,0.014858666807413101
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,7,0.013199999928474426
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,15,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,15,0.014826666563749313
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,15,0.013493333011865616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,15,0.014981333166360855
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,31,0.01312000056107839
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,31,0.015050667027632395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,31,0.014896000425020853
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,31,0.01309866706530253
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,63,0.015413332730531693
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,63,0.015253332753976187
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,63,0.01314666618903478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,63,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,127,0.013237333546082178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,127,0.013338666409254074
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,127,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,127,0.013381333400805792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,255,0.01488000030318896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,255,0.014970666418472925
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,255,0.01331199953953425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,255,0.016575999557971954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,511,0.01740266631046931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,511,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1023,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1023,0.02309333284695943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1023,0.01498666654030482
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1023,0.02143999934196472
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,2047,0.017231999586025875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,2047,0.039706667264302574
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,511,0.013717333475748697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,2047,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,2047,0.03170666595300039
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,4095,0.017488000293572743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,4095,0.06192533175150553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,4095,0.054133335749308266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,8191,0.01716800034046173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,8191,0.017093333105246227
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,3,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,4095,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,8191,0.08897599577903748
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,3,0.010629333555698395
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,7,0.010645333677530289
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,7,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,8191,0.10404800375302632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,63,0.010431999961535135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,63,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,255,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,255,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,511,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,511,0.010773333410422007
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,511,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1023,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1023,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1023,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,2047,0.013397333522637686
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,2047,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,4095,0.01292266696691513
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,4095,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,4095,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,4095,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,8191,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,8191,0.01718933383623759
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,8191,0.013248000293970108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,8191,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,16383,0.013584000368913015
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,16383,0.019333332777023315
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,16383,0.013183999806642532
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,16383,0.019088000059127808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,32767,0.02109333376089732
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,32767,0.012810666114091873
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,32767,0.021226666867733
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,65535,0.012896000097195307
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,65535,0.02513599892457326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,65535,0.013077333569526672
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,65535,0.02515200028816859
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,131071,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,131071,0.043194666504859924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,131071,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1,0.01748266691962878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1,0.01752000053723653
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,131071,0.03333866596221924
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,3,0.017749333133300144
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,3,0.016927999754746754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,3,0.01701333373785019
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,3,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,7,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,7,0.017045332739750545
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,7,0.016997333616018295
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,7,0.01708799973130226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,15,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,15,0.017312000195185345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,15,0.0170666662355264
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,15,0.0169813334941864
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,31,0.01732800031701724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,31,0.017018667111794155
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,31,0.016773333152135212
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,31,0.017130666722853977
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,63,0.01735466718673706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,63,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,63,0.01685333376129468
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,63,0.016965333372354507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,127,0.017008000363906223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1,0.017242666333913803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,127,0.017290666699409485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,255,0.01741333305835724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,255,0.017237332959969837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,255,0.01700266698996226
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,255,0.01728533332546552
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,127,0.017077332983414333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,511,0.019082666685183842
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,511,0.023237332701683044
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,511,0.01729600007335345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,511,0.021312000850836437
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,127,0.01706133286158244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1023,0.018986667195955913
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1023,0.017658667018016178
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1023,0.02940800040960312
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,2047,0.021514666577180225
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,2047,0.05813866853713989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,2047,0.020949333906173706
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,2047,0.0499893327554067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1023,0.034927998979886375
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,4095,0.02147199958562851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,4095,0.09569066762924194
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,4095,0.020143999407688778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,4095,0.08225599924723308
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1,0.025568000972270966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1,0.02367466688156128
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1,0.023242667317390442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,3,0.025231999655564625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,3,0.025445332129796345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,3,0.02367999901374181
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,3,0.023520000278949738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,7,0.025594666600227356
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,7,0.025125332176685333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,7,0.02365333338578542
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,7,0.025029333929220837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,15,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,15,0.025226667523384094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,15,0.02476799984773
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,15,0.023071999351183575
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,31,0.025455998877684276
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,31,0.024469333390394848
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,31,0.02330133318901062
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,63,0.025199999411900837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,63,0.025439999997615814
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,63,0.02313599983851115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,63,0.02346133440732956
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,127,0.025519999365011852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,127,0.02518400053183238
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,127,0.023472001155217487
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,127,0.02388266722361247
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,31,0.026208000878492992
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,255,0.0258240004380544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,255,0.02513599892457326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,255,0.02513599892457326
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,511,0.025765334566434223
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,255,0.025253333151340485
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,511,0.03748266647259394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,511,0.02533866713444392
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,511,0.03162666658560435
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1023,0.025685332715511322
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1023,0.025381334125995636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1023,0.04994133114814758
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,2047,0.029370665550231934
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,2047,0.09709866841634114
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1,0.01071999967098236
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,2047,0.027744000156720478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,2047,0.08494933446248372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,15,0.012298667182525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,15,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,127,0.010762666662534079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,511,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,511,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,511,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1023,0.010784000158309937
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1023,0.010869332899649939
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,2047,0.012800000607967377
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1023,0.05632533133029938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,2047,0.01479999969402949
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,4095,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,4095,0.01664000004529953
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,4095,0.013002666334311167
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,4095,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,8191,0.012821332861979803
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,8191,0.01695999999841054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,8191,0.013045333325862885
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,8191,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,16383,0.012794667234023413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,16383,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,32767,0.012997332960367203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,32767,0.025279998779296875
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,32767,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,32767,0.023408000667889912
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,16383,0.020096000283956528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,16383,0.021007999777793884
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,65535,0.012879999975363413
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,65535,0.040031999349594116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,65535,0.01303999995191892
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,65535,0.03143466760714849
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,131071,0.014949332922697067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,131071,0.06108800073464712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,131071,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,3,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,131071,0.0540533314148585
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,127,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,7,0.011333333949247995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,511,0.011626667032639185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,511,0.01098666712641716
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1023,0.012960000584522883
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,2047,0.017055999487638474
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,2047,0.015301333119471868
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,4095,0.018965333700180054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,4095,0.017029333859682083
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,8191,0.022645334402720135
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,8191,0.01897066707412402
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,16383,0.041450666884581246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,15,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,16383,0.023936000963052113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,32767,0.061237335205078125
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,32767,0.03972266614437103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,65535,0.09963732957839966
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,65535,0.05893866717815399
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,3,0.010634666929642359
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,3,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,7,0.011296000331640244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,131071,0.1853653391202291
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,7,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,15,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,15,0.011519999553759893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,63,0.011322667201360067
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,127,0.011381333072980246
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,255,0.011535999675591787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,511,0.012778667112191519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1023,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,31,0.01062400018175443
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,2047,0.011205332974592844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,2047,0.01139733319481214
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,4095,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,4095,0.011488000551859537
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,8191,0.015125333021084467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,8191,0.015263999501864115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,16383,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,131071,0.10084266463915507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1023,0.011312000453472137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,32767,0.027461332579453785
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,65535,0.03140799949566523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,65535,0.030810666580994923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,31,0.011557333171367645
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,32767,0.027376001079877216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,3,0.012373333175977072
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,16383,0.019285333653291065
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,131071,0.03398933261632919
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,131071,0.033530667424201965
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,511,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1023,0.010837333897749582
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,2047,0.012981332838535309
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,2047,0.013023999830087027
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,4095,0.013386666774749756
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,4095,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,8191,0.014912000546852747
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,8191,0.017050666113694508
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,16383,0.019648000597953796
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,16383,0.019359999646743137
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,32767,0.02346666653951009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,32767,0.023226665953795116
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,65535,0.0281333327293396
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,131071,0.0305173322558403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,3,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,7,0.011413333316644033
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,15,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,131071,0.027679999669392902
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,63,0.010538666198650995
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,63,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,511,0.013061333447694778
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,511,0.012773333738247553
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,2047,0.017162666966517765
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,65535,0.02532800038655599
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,2047,0.017071999609470367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,4095,0.021146667500336964
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,8191,0.039674667020638786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1023,0.012853333105643591
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,8191,0.02372266600529353
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,4095,0.019029332945744198
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,16383,0.061610668897628784
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,16383,0.0395359992980957
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,32767,0.10123200217882793
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,32767,0.06001600126425425
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,65535,0.1828320026397705
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,65535,0.09878399968147278
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1,0.01146666705608368
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,15,0.011029332876205444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,131071,0.3481280008951823
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,255,0.01073066641887029
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,131071,0.18112534284591675
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,511,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1023,0.01081066702802976
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1023,0.011349332829316458
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,4095,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,4095,0.013194666554530462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,8191,0.01903466631968816
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,8191,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,16383,0.021045332153638203
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,16383,0.020645332833131153
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,32767,0.024842667082945507
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,32767,0.021333334346612293
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,2047,0.015002666662136713
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,65535,0.02510400116443634
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1,0.011231999844312668
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,3,0.01128000020980835
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,131071,0.02938133229811986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,131071,0.04748799900213877
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,127,0.011226666470368704
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,255,0.011328000575304031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,511,0.013066666821638743
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1023,0.015146666516860327
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1023,0.013151999562978745
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,2047,0.02107733239730199
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,2047,0.018746666610240936
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,4095,0.03955733279387156
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,4095,0.023552000522613525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,8191,0.06042666733264923
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,8191,0.03976000100374222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,65535,0.023552000522613525
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,16383,0.060565332571665444
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,32767,0.1821333368619283
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,32767,0.0995360016822815
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1,0.012944000462690989
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,3,0.013013333082199097
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,3,0.01249066616098086
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,16383,0.10110933581988017
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,7,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,65535,0.3450400034586589
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,7,0.01259200026591619
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,15,0.012906666845083237
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,31,0.013381333400805792
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,65535,0.17897599935531616
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,63,0.012991999586423239
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,63,0.012736000120639801
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,127,0.013418667018413544
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,127,0.012928000340859095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,255,0.012885333349307379
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,511,0.015189333508412043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,511,0.013280000537633896
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,31,0.0129120002190272
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1023,0.019178666174411774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1023,0.018730666488409042
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,2047,0.03968533376852671
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,2047,0.025424001117547352
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,4095,0.06213866670926412
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,15,0.013477332890033722
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,8191,0.10307733217875163
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,8191,0.061434666315714516
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,4095,0.04014399896065394
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1,0.014965333044528961
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1,0.015103999525308609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,3,0.015178666760524115
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,3,0.015008000036080679
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,7,0.015040000279744467
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,7,0.015376000354687372
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,16383,0.10296533505121867
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,15,0.014869333555301031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,31,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,31,0.014842666685581207
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,63,0.014975999792416891
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,63,0.015141333142916361
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,127,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,127,0.015061333775520325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,255,0.014943999548753103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,255,0.014906667172908783
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,511,0.019120000302791595
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,511,0.017557332913080852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,16383,0.18545067310333252
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1023,0.03391999999682108
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1023,0.02128000060717265
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,15,0.014959999670584997
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,2047,0.06154666841030121
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,2047,0.041477332512537636
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,4095,0.10419199864069621
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,4095,0.06257066627343495
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1,0.011168000598748526
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,7,0.012506666282812754
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,8191,0.18702934185663858
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,127,0.01118933285276095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,255,0.011264000087976456
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1023,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,2047,0.012890666723251343
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,4095,0.01710933322707812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,4095,0.015168000012636185
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,8191,0.01893866683046023
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,8191,0.01836799954374631
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,8191,0.10408000151316325
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,2047,0.013855999956528345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,16383,0.019365333020687103
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,32767,0.025146665672461193
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,32767,0.021114667256673176
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,65535,0.04197333256403605
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,15,0.010751999914646149
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,131071,0.06392533580462138
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,131071,0.04260799785455068
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,16383,0.021381333470344543
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1,0.0194560003777345
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1,0.017173333714405697
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,3,0.01937066639463107
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,3,0.0173333336909612
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,7,0.019071999937295914
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,7,0.018794666975736618
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,15,0.01926933353145917
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,65535,0.025519999365011852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,31,0.019343999524911244
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,31,0.01740266631046931
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,63,0.01937599976857503
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,63,0.01842133328318596
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,127,0.01934933289885521
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,127,0.017360000560681026
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,255,0.019141333798567455
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,15,0.01854933301607768
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,511,0.033386667569478355
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,511,0.02325333406527837
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1023,0.052527998884518944
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1023,0.03401066611210505
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,255,0.017221332838137943
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,2047,0.0969599982102712
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,2047,0.05856533348560333
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1,0.029525332152843475
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,4095,0.17520000537236533
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,4095,0.0960586667060852
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,3,0.02939733366171519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,3,0.02568000058333079
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,7,0.029535998900731403
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,7,0.025311999022960663
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1,0.02589333305756251
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,15,0.02956799914439519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,15,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,31,0.029285334050655365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,31,0.025413334369659424
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,63,0.02924266705910365
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,63,0.025616000096003216
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,127,0.029663999875386555
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,127,0.025744001070658367
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,255,0.033610666791598
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,255,0.02537599951028824
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,511,0.05484800040721893
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,511,0.03766400118668874
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1023,0.09291199843088786
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1,0.011429333438475927
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,3,0.011535999675591787
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,2047,0.17466133832931519
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,31,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1023,0.055904000997543335
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,63,0.011343999455372492
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,255,0.011338666081428528
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,511,0.011247999966144562
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1023,0.011477333803971609
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,2047,0.015247999380032221
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,2047,0.01516266663869222
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,4095,0.017370666066805523
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,8191,0.01978133370478948
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,8191,0.0189280000825723
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,16383,0.025663999219735462
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,16383,0.02147199958562851
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,4095,0.01710933322707812
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,2047,0.09719467163085938
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,32767,0.02536533276240031
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,65535,0.0601440022389094
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,65535,0.04038933416207632
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,32767,0.04141866664091746
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,131071,0.10085866848627727
TRTLLM,1.2.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,131071,0.0613919993241628
