framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,1,0.012969599664211273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,1,0.012411200255155564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,1,0.012132800370454788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,1,0.012068799883127212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,1,0.012036799639463424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,1,0.012624000012874604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,1,0.018369600176811218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,1,0.017822399735450745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,1,0.01818400025367737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,1,0.017820799350738527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,1,0.017795200645923614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,1,0.01789119988679886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,1,0.017926399409770966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,3,0.012777599692344665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,3,0.012532800436019897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,3,0.012176000326871873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,3,0.012238399684429168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,3,0.012127999961376191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,3,0.011959999799728394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,3,0.018646399676799773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,3,0.01797119975090027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,3,0.018028800189495087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,3,0.017772799730300902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,1,0.012371200323104858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,7,0.01255040019750595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,7,0.012104000151157378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,7,0.01210239976644516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,7,0.012097600102424621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,7,0.012070400267839431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,7,0.01863519996404648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,7,0.018087999522686006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,7,0.017960000038146972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,7,0.017884799838066102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,7,0.017824000120162962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,7,0.017820799350738527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,15,0.012942400574684144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,15,0.012601600587368011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,15,0.01242239996790886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,15,0.012273599952459335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,15,0.012163200229406358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,15,0.012241599708795547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,15,0.01218239963054657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,15,0.018615999817848207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,15,0.018353599309921264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,15,0.018113599717617036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,15,0.01799360066652298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,15,0.01793760061264038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,15,0.01790879964828491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,15,0.01801919937133789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,31,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,31,0.012452799826860428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,3,0.01785600036382675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,31,0.012353599816560746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,31,0.012227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,31,0.012251199781894683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,31,0.012217599898576736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,31,0.018662400543689728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,31,0.018156799674034118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,31,0.018052799999713896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,31,0.018052799999713896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,31,0.01801439970731735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,31,0.01793439984321594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,31,0.018011200428009033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,63,0.013014400005340576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,63,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,63,0.012425599992275238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,63,0.01223360002040863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,63,0.012201599776744843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,63,0.012256000190973282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,63,0.012273599952459335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,63,0.018836799263954162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,63,0.01844319999217987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,63,0.01806560009717941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,63,0.01807360053062439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,63,0.018143999576568603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,63,0.01810719966888428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,63,0.018049600720405578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,127,0.014771200716495514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,127,0.014312000572681427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,127,0.014134399592876434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,31,0.012348800152540206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,127,0.014102399349212646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,127,0.014120000600814819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,127,0.01406559944152832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,127,0.020871999859809875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,127,0.020033599436283113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,127,0.019859200716018675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,127,0.019934399425983428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,127,0.019832000136375427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,127,0.019782400131225585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,127,0.01973759979009628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,255,0.017817600071430205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,255,0.017507199943065644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,255,0.01732639968395233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,255,0.017164799571037292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,255,0.017159999907016756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,255,0.017195199429988862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,255,0.0171424001455307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,255,0.02356639951467514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,255,0.02314079999923706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,3,0.017827199399471284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,255,0.022944000363349915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,255,0.022622400522232057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,255,0.022732800245285033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,255,0.02290080040693283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,255,0.022801600396633148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,511,0.018644799292087556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,511,0.017377600073814392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,511,0.01666239947080612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,511,0.016329599916934966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,511,0.017459200322628023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,511,0.017683200538158417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,511,0.01767520010471344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,511,0.02473440021276474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,511,0.0237744003534317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,511,0.022935999929904936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,511,0.022515200078487396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,511,0.023688000440597535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,511,0.02401760071516037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,511,0.023871999979019166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,1023,0.01902720034122467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,7,0.012929600477218629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,1023,0.017852799594402315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,3,0.018025599420070648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,1023,0.016689600050449373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,1023,0.017824000120162962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,1023,0.017292800545692443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,1023,0.017836800217628478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,1023,0.017825600504875184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,7,0.012486399710178375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,1023,0.024430400133132933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,1023,0.025596800446510314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,1023,0.023676800727844238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,1023,0.02414720058441162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,1023,0.0243136003613472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,2047,0.01972000002861023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,2047,0.01870879977941513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,2047,0.017888000607490538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,1023,0.02404959946870804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,2047,0.01794240027666092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,2047,0.018475200235843658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,2047,0.017961600422859193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,2047,0.02871200144290924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,2047,0.027116799354553224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,2047,0.0267984002828598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,2047,0.025824001431465148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,2047,0.026902401447296144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,2047,0.026080000400543212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,2047,0.026859200000762938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,4095,0.022038400173187256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,4095,0.01977760046720505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,4095,0.019280000030994414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,4095,0.020107200741767882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,4095,0.0202224001288414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,4095,0.020187200605869295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,4095,0.03360159993171692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,4095,0.03194240033626557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,4095,0.031176000833511353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,4095,0.030604800581932066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,4095,0.03138239979743958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,4095,0.031753599643707275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,4095,0.03176639974117279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,3,0.01218239963054657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,8191,0.025183999538421632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,8191,0.02195200026035309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,8191,0.02163680046796799
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,7,0.01820639967918396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,127,0.014100800454616546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,1023,0.024369600415229797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,8191,0.024111999571323393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,8191,0.0440095990896225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,2047,0.017774400115013123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,8191,0.04066559970378876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,8191,0.039945599436759946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,8191,0.039444801211357114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,8191,0.04132480025291443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,8191,0.04143519997596741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,16383,0.03094879984855652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,16383,0.025516799092292784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,16383,0.026009601354599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,16383,0.027515199780464173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,16383,0.025814399123191833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,16383,0.02619200050830841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,16383,0.06606720089912414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,16383,0.06136000156402588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,16383,0.05735840201377869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,16383,0.05617759823799133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,16383,0.056494402885437014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,16383,0.05652959942817688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,16383,0.05620800256729126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,32767,0.04254400134086609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,32767,0.0350271999835968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,32767,0.031672000885009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,32767,0.031121599674224853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,32767,0.030487999320030212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,32767,0.030695998668670656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,32767,0.10224000215530396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,8191,0.023555199801921844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,32767,0.09960799813270568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,8191,0.023520000278949738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,32767,0.09810240268707275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,32767,0.09863200187683105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,8191,0.024073599278926848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,32767,0.0981935977935791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,32767,0.09749280214309693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,8191,0.04071199893951416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,65535,0.051211202144622804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,16383,0.02526400089263916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,65535,0.05130079984664917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,65535,0.050083202123641965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,65535,0.05085279941558838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,4095,0.02086080014705658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,65535,0.05109279751777649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,65535,0.17566239833831787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,65535,0.17287360429763793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,65535,0.1723407983779907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,65535,0.1703552007675171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,65535,0.17138400077819824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,65535,0.17135839462280272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,131071,0.0900223970413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,131071,0.08302720189094544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,4,131071,0.08081920146942138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,8,131071,0.07894399762153625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,16,131071,0.0773248016834259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,32,131071,0.07775520086288452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1,1,64,131071,0.07803040146827697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,1,131071,0.3182143926620483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,2,131071,0.3155855894088745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,4,131071,0.3142927885055542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,8,131071,0.31341919898986814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,16,131071,0.3126591920852661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,32767,0.09813600182533264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,32,131071,0.3133951902389526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,1,0.012907199561595917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,1,0.012540799379348756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,1,0.012200000137090683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,1,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,1,0.012100800126791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,1,0.012115199863910676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,1,0.018371200561523436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,1,0.017790399491786957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,1,0.01762399971485138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,1,0.017628799378871917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,1,0.01769919991493225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,1,0.01770399957895279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,1,0.01756319999694824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,3,0.012822400033473968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,65535,0.17011200189590453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,3,0.012329600006341934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,3,0.012281599640846252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,3,0.012068799883127212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,3,0.012185599654912949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,3,0.012267199903726577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,3,0.018376000225543976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,3,0.01794400066137314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,3,0.017641599476337432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,3,0.017617599666118623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,3,0.017718400061130523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,3,0.01764959990978241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,3,0.017582400143146514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,7,0.01292639970779419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,7,0.01250080019235611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,7,0.012272000312805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,7,0.012265600264072418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,7,0.012263999879360199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,1,65535,0.05975679755210876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,7,0.012257599830627441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1,1,64,131071,0.31291201114654543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,7,0.01839520037174225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,65535,0.05584959983825684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,7,0.017579199373722078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,7,0.01793439984321594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,7,0.017523199319839478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,7,0.01761920005083084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,7,0.0176256000995636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,7,0.017601600289344786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,15,0.012958399951457977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,15,0.01239520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,15,0.012476799637079239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,15,0.012318400293588638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,15,0.012230399996042252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,15,0.01226079985499382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,15,0.018681600689888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,15,0.017876799404621124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,15,0.017844800651073457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,15,0.01767680048942566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,15,0.017817600071430205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,15,0.01765120029449463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,31,0.013006399571895599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,31,0.012539200484752655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,31,0.01236959993839264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,31,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,31,0.012164799869060517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,31,0.012160000205039979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,3,0.012108799815177918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,31,0.01841440051794052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,31,0.017969599366188048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,31,0.01780959963798523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,31,0.017612800002098083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,31,0.017803199589252472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,31,0.017561599612236023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,31,0.017617599666118623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,63,0.013020800054073333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,63,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,63,0.012254399806261062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,63,0.012111999839544297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,63,0.012118399888277055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,63,0.012163200229406358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,63,0.012252800166606903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,63,0.0185248002409935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,63,0.018062399327754976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,63,0.01780160069465637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,63,0.01770240068435669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,7,0.012095999717712403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,1,0.012211199849843979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,63,0.017847999930381775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,127,0.014800000190734863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,63,0.018030400574207305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,127,0.014371199905872345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,127,0.014187200367450714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,127,0.014057600498199463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,127,0.014124800264835358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,127,0.014001600444316864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,127,0.013990400731563568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,15,0.018059200048446654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,127,0.020257599651813507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,127,0.019964799284934998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,127,0.019555200636386872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,127,0.01971199959516525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,127,0.019392000138759614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,127,0.01972000002861023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,255,0.01777919977903366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,255,0.017508800327777862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,255,0.017159999907016756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,31,0.01210559979081154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,255,0.017113600671291352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,2,32767,0.0383215993642807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,255,0.01706400066614151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,255,0.017080000042915343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,255,0.022951999306678773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,255,0.02276960015296936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,255,0.022811199724674224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,255,0.02264160066843033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,255,0.02290239930152893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,255,0.02260800004005432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,511,0.01895360052585602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,511,0.017972800135612487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,511,0.017124800384044646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,511,0.01783040016889572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,511,0.01799200028181076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,511,0.017987200617790224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,511,0.025385600328445435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,511,0.02407840043306351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,511,0.023201599717140198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,511,0.022987200319766997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,511,0.024057599902153014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,511,0.024270400404930115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,63,0.017820799350738527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,511,0.024255999922752382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,1023,0.019417600333690645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,1023,0.018167999386787415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,1023,0.017419199645519256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,1023,0.01720000058412552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,1023,0.017833599448204042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,1023,0.018185600638389587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,15,0.012212800234556198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,1023,0.018089599907398224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,1023,0.027884799242019653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,1023,0.026199999451637267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,1023,0.02502560019493103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,1023,0.025736001133918763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,127,0.019539199769496918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,1023,0.025960001349449157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,1023,0.025942400097846985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,2047,0.020532800257205962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,2047,0.018334400653839112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,2047,0.018182399868965148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,2047,0.018275199830532073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,2047,0.018500800430774688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,2047,0.01823839992284775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,2047,0.03203519880771637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,2047,0.02998400032520294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,2047,0.02943840026855469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,255,0.023582400381565095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,2047,0.028968000411987306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,2047,0.02935360074043274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,2047,0.0293503999710083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,2047,0.029516801238059998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,4095,0.020776000618934632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,4095,0.02008160054683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,4095,0.019542400538921357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,4095,0.020294399559497835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,4095,0.02048480063676834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,511,0.016780799627304076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,4095,0.037862399220466615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,4095,0.03722079992294312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,4095,0.036801600456237794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,4095,0.03725599944591522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,4095,0.03766080141067505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,4095,0.037699198722839354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,8191,0.027662399411201476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,8191,0.02399519979953766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,8191,0.023201599717140198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,8191,0.022339199483394623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,8191,0.025176000595092774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,8191,0.025513601303100587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,8191,0.025620800256729127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,8191,0.06338719725608825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,8191,0.05745760202407837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,8191,0.05437279939651489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,1023,0.025672000646591187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,8191,0.0523792028427124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,8191,0.055576002597808837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,8191,0.05524160265922547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,2047,0.019203199446201323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,8191,0.05648159980773926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,16383,0.03535839915275574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,16383,0.029252800345420837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,16383,0.027113598585128785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,16383,0.02969599962234497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,16383,0.02958720028400421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,16383,0.03019680082798004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,16383,0.09917280077934265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,16383,0.0956175982952118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,16383,0.09485120177268982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,16383,0.09416319727897644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,4095,0.020545600354671477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,16383,0.09630240201950073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,16383,0.09662240147590637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,16383,0.09779199957847595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,32767,0.05607200264930725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,32767,0.05080159902572632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,32767,0.047443199157714847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,32767,0.04930559992790222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,32767,0.04929440021514893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,32767,0.04933919906616211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,32767,0.1727023959159851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,32767,0.16958880424499512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,32767,0.1671519994735718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,32767,0.16810879707336426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,32767,0.16898560523986816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,32767,0.16960959434509276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,32767,0.16922880411148072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,65535,0.08442080020904541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,65535,0.07890719771385193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,65535,0.07378720045089722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,65535,0.07517759799957276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,65535,0.07542240023612976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,16383,0.039094400405883786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,65535,0.07560480237007142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,255,0.017192000150680543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,65535,0.31548318862915037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,65535,0.31181600093841555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,65535,0.3096784114837646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,65535,0.3085808038711548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,65535,0.3106208086013794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,32767,0.04834559857845307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,65535,0.31070239543914796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,131071,0.1372704029083252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,2,131071,0.12999680042266845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,131071,0.12716319561004638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,8,131071,0.12615360021591188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,16,131071,0.126803195476532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,32,131071,0.1264863967895508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,2,1,64,131071,0.12632160186767577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,131071,0.5924767971038818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,2,131071,0.5862832069396973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,4,65535,0.07560960054397584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,4,131071,0.586510419845581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,8,131071,0.5824992179870605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,1,0.013204799592494964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,1,0.012574400007724761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,131071,0.586510419845581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,32,131071,0.5843967914581298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,2,1,64,131071,0.5840943813323974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,1,0.012217599898576736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,1,0.012204799801111221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,1,0.0182096004486084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,1,0.01793439984321594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,1,4095,0.02306720018386841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,1,0.01793919950723648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,1,0.01806560009717941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,1,0.01810719966888428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,1,4095,0.04070079922676086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,3,0.013012799620628356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,3,0.012595200538635254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,3,0.012212800234556198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,3,0.012342400103807449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,3,0.012268800288438797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,3,0.012147200107574464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,3,0.01855839937925339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,3,0.018190400302410127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,3,0.018140800297260284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,3,0.017895999550819396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,3,0.017884799838066102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,3,0.01778720021247864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,3,0.01781439930200577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,7,0.012929600477218629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,7,0.01266240030527115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,7,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,1,0.012537600100040435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,1,0.012347199767827988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,7,0.012326399981975555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,1,0.01228799968957901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,7,0.012238399684429168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,1,0.01871200054883957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,7,0.01802240014076233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,7,0.017849600315093993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,7,0.017880000174045563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,7,0.017868800461292265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,1,0.01788640022277832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,7,0.017752000689506532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,7,0.017774400115013123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,15,0.012969599664211273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,15,0.012443199753761292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,15,0.012276799976825714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,15,0.012166400253772736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,15,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,15,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,15,0.012268800288438797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,15,0.018884800374507904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,15,0.01817599982023239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,15,0.01823839992284775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,15,0.017811200022697447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,15,0.017948800325393678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,15,0.017972800135612487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,15,0.018035200238227845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,31,0.013049599528312684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,31,0.012558400630950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,31,0.012342400103807449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,31,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,31,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,31,0.012193600088357926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,31,0.012153600156307221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,31,0.018632000684738158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,31,0.018171200156211854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,31,0.01799360066652298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,31,0.017878399789333345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,31,0.018062399327754976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,31,0.017969599366188048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,31,0.018080000579357148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,63,0.01311040073633194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,63,0.012513600289821625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,63,0.01249919980764389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,63,0.012283200025558471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,63,0.012329600006341934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,63,0.012251199781894683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,63,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,63,0.01868959963321686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,63,0.018198400735855103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,63,0.018198400735855103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,63,0.017980800569057466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,63,0.018118399381637573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,63,0.018089599907398224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,63,0.018104000389575957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,127,0.014747199416160584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,127,0.014204800128936768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,127,0.01412159949541092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,127,0.013963200151920319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,127,0.013966399431228637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,127,0.013953599333763122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,127,0.013867199420928955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,127,0.020503999292850496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,127,0.020001600682735442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,127,0.019952000677585603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,127,0.019739200174808503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,127,0.019838400185108185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,127,0.01988160014152527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,127,0.019721600413322448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,16,65535,0.31045920848846437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,255,0.017287999391555786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,255,0.017235200107097625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,3,0.012247999757528305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,255,0.01706880033016205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,255,0.01706079989671707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,255,0.023892800509929656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,255,0.023499199748039247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,255,0.023315200209617616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,255,0.023209600150585173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,255,0.023244799673557283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,255,0.023233599960803986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,255,0.023095999658107758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,511,0.019336000084877014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,511,0.018108800053596497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,511,0.017444799840450286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,511,0.016990399360656737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,511,0.018119999766349794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,511,0.01831679940223694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,511,0.018371200561523436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,7,0.012403199821710587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,511,0.0279664009809494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,511,0.02688480019569397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,511,0.02540160119533539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,7,0.018614399433135986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,511,0.026311999559402464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,511,0.026528000831604004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,511,0.026480001211166383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,1023,0.019827200472354888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,1023,0.01839359998703003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,1023,0.018119999766349794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,1023,0.01743679940700531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,1023,0.01857600063085556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,255,0.01780640035867691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,1023,0.018379199504852294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,1023,0.03101919889450073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,1023,0.02967039942741394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,1023,0.028575998544692994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,255,0.017102399468421937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,1023,0.029073598980903625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,1023,0.02935839891433716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,1023,0.029390400648117064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,2047,0.022996799647808076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,2047,0.020043200254440306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,2047,0.018982400000095368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,2047,0.018324799835681915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,2047,0.018926399946212768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,2047,0.01980320066213608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,2047,0.018875199556350707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,2047,0.03986240029335022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,2047,0.037248000502586365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,2047,0.036155200004577635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,2047,0.03567039966583252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,2047,0.036048001050949095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,2047,0.03596799969673157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,2047,0.03613759875297547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,4095,0.022566400468349457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,4095,0.02207999974489212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,4095,0.021249599754810333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,4095,0.02250239998102188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,4095,0.02235199958086014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,4095,0.022787199914455415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,511,0.02574400007724762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,4095,0.06175839900970459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,7,0.012216000258922577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,4095,0.05605120062828064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,4095,0.051712000370025636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,4095,0.053401601314544675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,4095,0.053651201725006106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,4095,0.05355200171470642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,8191,0.037062400579452516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,1023,0.018406400084495546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,8191,0.028479999303817748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,8191,0.026657599210739135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,8191,0.029187199473381043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,255,0.01698880046606064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,8191,0.03027839958667755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,1023,0.028435200452804565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,8191,0.03052319884300232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,8191,0.09446560144424439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,8191,0.09731040000915528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,8191,0.09225280284881592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,8191,0.09700639843940735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,8191,0.09691680073738099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,8191,0.09682559967041016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,16383,0.0491344004869461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,16383,0.047009599208831784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,16383,0.04551999866962433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,16383,0.04872959852218628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,16383,0.04915040135383606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,16383,0.049353599548339844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,16383,0.1714367985725403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,4095,0.0253248006105423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,16383,0.16787519454956054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,16383,0.1662224054336548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,16383,0.16509439945220947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,16383,0.16919360160827637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,16383,0.16958719491958618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,16383,0.16876800060272218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,32767,0.08375999927520753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,32767,0.07445279955863952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,32767,0.07321919798851013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,32767,0.076009601354599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,32767,0.07571679949760438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,4095,0.05208960175514221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,32767,0.07605119943618774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,32767,0.3134335994720459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,32767,0.31010239124298095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,32767,0.30687038898468016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,8191,0.03314239978790283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,32767,0.3061487913131714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,32767,0.3094799995422363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,32767,0.30969278812408446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,32767,0.3102911949157715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,65535,0.12482559680938721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,8191,0.09300479888916016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,65535,0.12331680059432984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,16383,0.05559999942779541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,65535,0.12616319656372071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,65535,0.12646399736404418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,65535,0.5875855922698975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,65535,0.5868080139160157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,65535,0.5837488174438477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,32767,0.07468799948692321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,65535,0.5865280151367187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,131071,0.23589119911193848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,2,131071,0.2313136100769043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,65535,0.5851103782653808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,131071,0.22727360725402831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,8,131071,0.22599680423736573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,131071,0.2279599905014038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,1,65535,0.13419359922409058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,16,131071,0.22806720733642577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,4,65535,0.12420480251312256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,32,65535,0.12634079456329345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,65535,0.5826384067535401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,65535,0.5802720069885254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,1,131071,1.1373647689819335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,2,131071,1.1355600357055664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,1,0.013017599284648896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,4,131071,1.1297295570373536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,8,131071,1.128934383392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,1,0.012604799866676331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,1,0.01242400035262108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,1,0.01226079985499382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,1,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,1,0.012230399996042252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,1,0.018748800456523895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,1,0.018484799563884734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,1,0.01815840005874634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,1,0.018172800540924072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,1,0.018163199722766876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,1,0.018287999927997588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,1,0.018164800107479097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,3,0.013075199723243714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,32,131071,1.130900764465332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,4,1,64,131071,1.130470371246338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,3,0.012382400035858155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,3,0.012436799705028534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,3,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,3,0.012462399899959564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,3,0.01902880072593689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,3,0.018427200615406036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,3,0.018140800297260284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,3,0.018129600584506987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,3,0.0182559996843338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,3,0.01833280026912689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,7,0.013156799972057343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,7,0.012681600451469422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,7,0.01247519999742508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,7,0.012268800288438797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,7,0.012430399656295776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,7,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,7,0.012408000230789185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,7,0.019073599576950075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,7,0.018624000251293182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,7,0.018371200561523436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,7,0.018328000605106354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,7,0.01831520050764084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,7,0.01820479929447174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,1,0.012273599952459335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,7,0.01826400011777878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,15,0.013118399679660797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,15,0.012839999794960023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,15,0.012455999851226807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,15,0.012451200187206269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,15,0.012366399914026261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,15,0.012465599924325943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,15,0.012348800152540206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,15,0.019179199635982514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,15,0.01830720007419586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,3,0.012566399574279786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,15,0.01831520050764084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,15,0.01815200001001358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,3,0.012289600074291229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,15,0.01812800019979477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,31,0.01311360001564026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,31,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,3,0.01873600035905838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,31,0.012451200187206269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,31,0.012374400347471236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,31,0.012638400495052337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,31,0.012334399670362473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,31,0.019180800020694732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,31,0.018620799481868743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,31,0.018433600664138794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,4,1,64,131071,0.22803199291229248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,31,0.018217599391937254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,31,0.018432000279426576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,31,0.018084800243377684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,31,0.018486399948596955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,63,0.012763200700283051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,63,0.01265919953584671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,63,0.01249919980764389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,63,0.012619200348854064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,63,0.012531200051307678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,63,0.012628799676895142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,63,0.01929599940776825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,63,0.018531200289726258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,63,0.018380799889564516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,63,0.018430399894714355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,63,0.018561600148677825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,63,0.018545599281787874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,127,0.014555199444293976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,127,0.014401599764823914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,127,0.014239999651908874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,127,0.01424960047006607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,127,0.014244799315929414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,127,0.014140799641609192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,127,0.021163199841976166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,127,0.02077919989824295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,16,131071,1.1316847801208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,127,0.02057439982891083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,15,0.01828159987926483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,127,0.02067680060863495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,127,0.020521600544452668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,127,0.020713600516319274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,127,0.0204815998673439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,31,0.012337599694728852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,255,0.017603200674057008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,255,0.017284800112247468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,255,0.01722719967365265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,255,0.017286400496959686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,255,0.017348800599575043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,255,0.017257599532604216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,255,0.026076799631118773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,63,0.013400000333786011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,255,0.0255840003490448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,255,0.025468799471855163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,255,0.025484800338745117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,255,0.0253711998462677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,255,0.025388801097869874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,511,0.019996799528598785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,511,0.019020800292491914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,63,0.01887200027704239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,511,0.01751520037651062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,511,0.018507200479507446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,511,0.01880960017442703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,511,0.01886879950761795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,511,0.03151200115680695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,511,0.030112001299858093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,511,0.029179200530052185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,511,0.028758400678634645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,511,0.029743999242782593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,511,0.029902398586273193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,127,0.0149167999625206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,1023,0.0225615993142128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,1023,0.020431999862194062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,1023,0.01810719966888428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,1023,0.01889919936656952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,1023,0.01892160028219223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,1023,0.019145600497722626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,1023,0.04026240110397339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,15,0.01857919991016388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,1023,0.03686240017414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,1023,0.03621279895305633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,1023,0.035452800989151004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,1023,0.03675679862499237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,1023,0.03632799983024597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,1023,0.03676159977912903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,2047,0.026638400554656983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,2047,0.02229599952697754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,255,0.018211199343204497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,2047,0.020712000131607056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,2047,0.02099359929561615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,2047,0.02080959975719452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,2047,0.021160000562667848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,2047,0.062028801441192626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,2047,0.055739200115203856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,2047,0.05353279709815979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,2047,0.051553601026535036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,255,0.025209599733352663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,2047,0.051984000205993655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,2047,0.05150399804115295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,4095,0.037089601159095764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,511,0.01788160055875778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,4095,0.03152959942817688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,4095,0.025867199897766112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,4095,0.0271807998418808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,4095,0.027371200919151305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,4095,0.09696159958839416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,511,0.029916799068450926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,4095,0.09303359985351563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,1023,0.019475199282169342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,4095,0.09211840033531189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,4095,0.09158719778060913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,4095,0.0940559983253479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,4095,0.09382879734039307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,4095,0.09425280094146729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,8191,0.05403040051460266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,8191,0.049323201179504395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,8191,0.046188798546791074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,8191,0.04482080042362213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,8191,0.04924960136413574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,8191,0.04928160011768341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,8191,0.05027040243148804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,8191,0.17081600427627563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,8191,0.16651999950408936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,8191,0.16627039909362792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,8191,0.16554720401763917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,8191,0.16854239702224733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,8191,0.1690400004386902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,2047,0.021379199624061585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,16383,0.07652480006217957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,16383,0.0737551987171173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,16383,0.07491679787635804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,16383,0.07481279969215393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,16383,0.0758575975894928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,16383,0.312775993347168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,16383,0.3087791919708252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,4095,0.027795198559761047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,16383,0.30590879917144775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,4095,0.026553601026535034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,16383,0.30474560260772704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,16383,0.308785605430603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,16383,0.30993759632110596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,16383,0.3094144105911255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,32767,0.1340127944946289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,32767,0.1264863967895508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,32767,0.12416800260543823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,32767,0.12263360023498535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,32767,0.12571040391921998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,32767,0.1262768030166626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,32767,0.12408159971237183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,32767,0.5886127948760986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,32767,0.5818607807159424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,32767,0.5807231903076172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,8191,0.17059359550476075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,16383,0.08287199735641479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,16383,0.07015039920806884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,32767,0.5805024147033692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,32767,0.5816319942474365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,65535,0.23512959480285645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,32767,0.583299207687378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,32767,0.5864480018615723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,65535,0.22951679229736327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,65535,0.2283519983291626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,65535,0.224182391166687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,65535,0.2272655963897705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,65535,0.22828640937805175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,65535,1.1374287605285645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,65535,1.1253775596618651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,65535,1.1358287811279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,1,131071,0.43752322196960447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,65535,1.1319120407104493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,65535,1.1264479637145997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,2,131071,0.4316847801208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,2047,0.05248479843139649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,131071,0.4289087772369385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,8,131071,0.42704482078552247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,16,131071,0.4304031848907471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,32,131071,0.43146238327026365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,8,1,64,131071,0.4308479785919189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,65535,1.1314528465270997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,1,131071,2.2197807312011717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,2,131071,2.2151552200317384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,131071,2.2212783813476564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,1,0.014404800534248353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,8,131071,2.208415985107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,1,0.01300320029258728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,1,0.012545600533485413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,4,65535,0.22518560886383057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,1,0.01239679977297783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,1,0.019070400297641753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,1,0.012494400143623352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,1,0.018292799592018127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,16,131071,2.226371192932129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,1,0.018415999412536622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,1,0.018027199804782866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,1,0.018503999710083006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,1,0.018249599635601042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,1,0.018406400084495546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,3,0.014268800616264343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,3,0.01271039992570877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,3,0.012544000148773193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,3,0.012457600235939026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,3,0.012460800260305405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,3,0.012489599734544754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,32,131071,2.229915237426758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,3,0.01855680048465729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,3,0.01831520050764084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,3,0.018892799317836762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,4,65535,1.1246623992919922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,3,0.018353599309921264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,3,0.018185600638389587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,7,0.01343040019273758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,8,1,64,131071,2.209059143066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,3,0.018324799835681915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,7,0.01273760050535202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,7,0.012368000298738479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,7,0.012464000284671784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,7,0.012582400441169738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,7,0.0187376007437706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,7,0.01870400011539459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,7,0.018276800215244294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,7,0.018403199315071107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,7,0.018225599825382233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,7,0.018296000361442567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,7,0.018177600204944612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,15,0.014159999787807465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,15,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,15,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,15,0.012548799812793731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,15,0.01252480000257492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,15,0.012430399656295776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,15,0.018908800184726716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,15,0.018748800456523895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,15,0.018275199830532073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,15,0.018086400628089905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,1,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,15,0.018060800433158875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,3,0.012860800325870513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,15,0.018212799727916718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,31,0.013142399489879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,31,0.013107199966907502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,31,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,31,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,31,0.012583999335765839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,31,0.01242239996790886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,7,0.013051199913024902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,31,0.012433599680662155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,31,0.019310399889945984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,7,0.0125231996178627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,31,0.018854400515556334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,31,0.018529599905014037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,31,0.018713599443435668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,31,0.01857600063085556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,63,0.013648000359535218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,63,0.013649600744247436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,63,0.012862400710582733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,63,0.012894399464130402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,15,0.012695999443531036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,63,0.012804800271987915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,63,0.012721599638462066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,63,0.01968960016965866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,63,0.019256000220775605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,63,0.019142399728298187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,15,0.018140800297260284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,63,0.019062399864196777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,63,0.01858240067958832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,63,0.01882079988718033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,63,0.019377599656581878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,127,0.015070399641990662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,127,0.0146479994058609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,127,0.014494399726390838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,127,0.014908799529075622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,127,0.014372800290584565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,127,0.01433439999818802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,127,0.023155200481414794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,127,0.022868800163269042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,127,0.022540800273418427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,127,0.022460800409317017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,127,0.022375999391078948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,31,0.018508799374103546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,127,0.022339199483394623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,127,0.022411200404167175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,255,0.018500800430774688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,255,0.017990399897098542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,255,0.017641599476337432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,255,0.01769919991493225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,255,0.018068799376487733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,255,0.018119999766349794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,255,0.01810079962015152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,255,0.02905600070953369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,255,0.02865760028362274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,255,0.028262400627136232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,255,0.028244799375534056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,255,0.028303998708724975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,255,0.02816160023212433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,255,0.028244799375534056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,511,0.02242559939622879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,63,0.01289760023355484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,511,0.019944000244140624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,511,0.018905599415302277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,511,0.019726400077342988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,511,0.019508799910545348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,511,0.019465599954128266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,1,0.012998400628566742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,511,0.03689279854297638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,511,0.036113598942756654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,511,0.036595198512077334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,127,0.01604959964752197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,511,0.03668479919433594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,511,0.03643360137939453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,1023,0.027331200242042542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,1023,0.022334399819374084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,1023,0.02043360024690628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,1023,0.021187199652194975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,1023,0.021264000236988066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,1023,0.02146880030632019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,1023,0.062745600938797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,1023,0.05601440072059631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,1023,0.052820801734924316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,3,0.018649600446224213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,1023,0.05181760191917419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,1023,0.0515824019908905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,1023,0.05165600180625916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,2047,0.038020798563957216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,2047,0.03413600027561188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,2047,0.02842719852924347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,2047,0.025305598974227905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,2047,0.025969600677490233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,2047,0.025542399287223815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,31,0.0186271995306015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,2047,0.025905600190162657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,2047,0.0984495997428894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,2047,0.09247360229492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,2047,0.09387360215187072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,2047,0.09231359958648681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,2047,0.09238560199737549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,2047,0.09275519847869873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,2047,0.09311839938163757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,4095,0.05390560030937195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,4095,0.04866400063037872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,4095,0.04523679912090302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,4095,0.044091200828552245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,4095,0.04587680101394653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,4095,0.04657599925994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,4095,0.046014401316642764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,511,0.041284799575805664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,4095,0.17214879989624024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,4095,0.1667215943336487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,4095,0.1641152024269104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,4095,0.1645840048789978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,4095,0.1659168004989624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,4095,0.16594879627227782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,4095,0.1669360041618347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,8191,0.0796832025051117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,8191,0.07352160215377808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,8191,0.07115359902381897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,8191,0.06888960003852844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,8191,0.0733568012714386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,8191,0.0741216003894806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,8191,0.07442240118980407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,1023,0.05026879906654358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,8191,0.3108047962188721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,8191,0.30730879306793213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,8191,0.3042671918869019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,8191,0.3032416105270386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,8191,0.3080912113189697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,8191,0.3105871915817261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,8191,0.3090960025787354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,16383,0.12984960079193114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,16383,0.12351039648056031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,16383,0.11888959407806396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,16383,0.1234976053237915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,16383,0.1242527961730957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,16383,0.12403039932250977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,511,0.018555200099945067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,16383,0.5793839931488037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,16383,0.5772319793701172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,511,0.035062399506568906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,16383,0.576697587966919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,16383,0.5815023899078369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,16383,0.5805056095123291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,1023,0.020945599675178526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,32767,0.23087360858917236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,16383,0.5819519996643067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,32767,0.22395999431610109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,32767,0.21990559101104737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,32767,0.2228559970855713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,32767,0.22292640209197997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,32767,0.22317759990692138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,16383,0.12131199836730958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,32767,1.1288816452026367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,32767,1.1263039588928223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,32767,1.123356819152832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,32767,1.1215503692626954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,16383,0.5846960067749023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,65535,0.42995681762695315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,32767,1.1268400192260741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,32767,1.1287487983703612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,32767,1.123641586303711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,65535,0.42005281448364257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,65535,0.41731839179992675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,65535,0.4214208126068115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,65535,0.420531177520752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,65535,0.4216639995574951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,32767,0.21969120502471923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,65535,2.221108818054199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,65535,2.211409568786621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,65535,2.2084447860717775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,65535,2.2156496047973633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,65535,0.4444736003875732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,65535,2.209382438659668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,65535,2.2156160354614256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,1,131071,0.8279376029968262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,65535,2.2172143936157225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,2,131071,0.820025634765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,4,131071,0.814844799041748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,8,131071,0.8143648147583008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,16,131071,0.816312026977539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,32,131071,0.8172431945800781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,16,1,64,131071,0.8170368194580078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,1,0.014691199362277984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,1,0.013337600231170654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,1,0.013247999548912048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,1,0.013022400438785553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,1,0.013191999495029449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,1,0.013246400654315949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,1,0.013212800025939941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,1,0.020497600734233856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,1,0.019230400025844575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,1,0.01876160055398941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,1,0.018956799805164338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,1,0.018931199610233308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,1,0.01879359930753708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,1,0.01887200027704239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,3,0.014643199741840363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,3,0.013515199720859527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,3,0.013233600556850434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,1,131071,4.371870422363282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,4,131071,4.411446380615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,2,131071,4.404001617431641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,8,131071,4.378275299072266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,16,131071,4.399415969848633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,3,0.013449600338935852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,16,1,64,131071,4.414707183837891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,3,0.013152000308036805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,3,0.01927199959754944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,3,0.013150399923324585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,3,0.018787199258804323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,3,0.01923840045928955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,3,0.019065600633621217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,7,0.01483200043439865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,7,0.013820800185203552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,7,0.013596799969673157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,7,0.013068799674510957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,7,0.01342719942331314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,7,0.013352000713348388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,7,0.013303999602794648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,7,0.020499199628829956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,7,0.019228799641132353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,7,0.019196799397468566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,7,0.01897120028734207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,7,0.0191551998257637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,7,0.018969599902629853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,7,0.01892160028219223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,15,0.014920000731945039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,15,0.013600000739097595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,15,0.013547199964523315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,15,0.013118399679660797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,15,0.013230399787425995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,15,0.01319040060043335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,15,0.013359999656677246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,15,0.020526400208473204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,15,0.019555200636386872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,15,0.019280000030994414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,15,0.019489599764347075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,15,0.01913760006427765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,15,0.01926079988479614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,15,0.019415999948978423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,31,0.015115199983119965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,31,0.013799999654293061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,31,0.013446399569511413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,31,0.013492800295352936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,31,0.013513599336147309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,31,0.013318400084972381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,31,0.021118399500846863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,32,131071,4.382235336303711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,31,0.020076799392700195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,31,0.019593599438667297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,31,0.019819200038909912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,31,0.019500799477100372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,31,0.019472000002861024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,31,0.01942719966173172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,63,0.014287999272346497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,63,0.013945600390434265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,63,0.013758400082588195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,63,0.013964800536632538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,63,0.013764800131320953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,63,0.01385599970817566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,63,0.02343679964542389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,63,0.022224000096321105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,63,0.02190079987049103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,63,0.02170239984989166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,63,0.021883200109004974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,63,0.021796800196170807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,63,0.021638399362564086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,127,0.01711200028657913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,127,0.015964800119400026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,31,0.013286399841308593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,127,0.015703999996185304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,127,0.01534239947795868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,3,0.012942400574684144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,127,0.015620799362659454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,127,0.027628800272941588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,127,0.026464000344276428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,127,0.026185598969459534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,127,0.026063999533653258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,127,0.026208001375198364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,3,0.01880960017442703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,127,0.026174399256706237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,3,0.01910240054130554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,255,0.018908800184726716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,255,0.01906079947948456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,255,0.018606400489807128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,255,0.0187376007437706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,255,0.018769599497318268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,255,0.03666079938411713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,255,0.03536159992218017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,255,0.03496640026569366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,255,0.034860798716545106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,255,0.0350271999835968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,255,0.03499360084533691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,255,0.03482399880886078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,511,0.02905279994010925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,127,0.015731200575828552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,511,0.02122559994459152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,127,0.015561600029468537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,511,0.021704000234603883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,3,0.021055999398231506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,63,0.015723200142383577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,511,0.021768000721931458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,127,0.026532799005508423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,511,0.06414880156517029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,511,0.05753600001335144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,511,0.05321279764175415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,511,0.05167999863624573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,511,0.05189599990844727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,511,0.05235199928283692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,511,0.05168160200119019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,1023,0.042033600807189944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,1023,0.030063998699188233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,1023,0.024985599517822265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,1023,0.024355199933052064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,1023,0.024568000435829164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,1023,0.024240000545978545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,1023,0.10064799785614013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,1023,0.09423360228538513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,1023,0.09242720007896424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,511,0.023937599360942842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,1023,0.09148319959640502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,1023,0.09217600226402282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,511,0.020190399885177613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,1023,0.09248319864273072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,1023,0.09173920154571533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,2047,0.05716800093650818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,2047,0.049798399209976196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,2047,0.046910399198532106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,2047,0.04471679925918579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,2047,0.045244801044464114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,2047,0.17390559911727904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,255,0.02051679939031601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,255,0.018713599443435668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,2047,0.1679152011871338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,2047,0.16537599563598632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,2047,0.16477919816970826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,2047,0.164902400970459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,2047,0.16471999883651733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,2047,0.16566239595413207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,4095,0.08277919888496399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,4095,0.07461599707603454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,4095,0.07195199728012085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,4095,0.06919199824333191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,4095,0.07173759937286377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,4095,0.07162399888038636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,4095,0.0712768018245697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,4095,0.31343679428100585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,4095,0.3075295925140381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,4095,0.303110408782959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,4095,0.3072448015213013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,4095,0.3062495946884155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,4095,0.30745439529418944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,8191,0.13238240480422975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,8191,0.12485920190811158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,2047,0.04530879855155945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,2047,0.044833600521087646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,8191,0.11947360038757324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,8191,0.12422720193862916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,8191,0.12483359575271606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,511,0.021612800657749176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,8191,0.5887887954711915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,8191,0.5828383922576904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,8191,0.5786128044128418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,8191,0.5787055969238282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,8191,0.5841152191162109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,8191,0.5829967975616455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,8191,0.5823664188385009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,16383,0.23275840282440186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,16383,0.22064480781555176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,16383,0.21855199337005615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,16383,0.22301440238952636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,16383,0.22393760681152344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,16383,0.22285919189453124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,8191,0.12207039594650268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,8191,0.12492320537567139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,16383,1.1238975524902344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,16383,1.1232864379882812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,1023,0.03463839888572693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,16383,1.1225616455078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,16383,1.1257663726806642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,32767,0.43233118057250974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,4095,0.3049999952316284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,16383,1.126747226715088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,32767,0.4249392032623291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,16383,0.22409279346466066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,32767,0.4210976123809814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,32767,0.41702561378479003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,32767,0.4211743831634521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,32767,0.4239344120025635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,32767,0.4221695899963379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,16383,1.1355104446411133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,32767,2.225294494628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,32767,2.214975929260254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,32767,2.1994255065917967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,32767,2.2146223068237303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,32767,2.2163888931274416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,32767,2.203000068664551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,16383,1.1239824295043945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,1,65535,0.829470443725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,2,65535,0.8193519592285157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,4,65535,0.8139216423034668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,8,65535,0.816160011291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,16,65535,0.8177871704101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,32767,2.2150943756103514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,32,65535,0.8190976142883301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,32,1,64,65535,0.8181424140930176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,1,65535,4.373924636840821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,4,65535,4.395606231689453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,8,65535,4.373511886596679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,1,0.01600320041179657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,16,65535,4.387003326416016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,1,0.014507199823856353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,32,1,64,65535,4.382950210571289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,1,0.014131200313568116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,1,0.01406719982624054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,1,0.014177599549293518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,1,0.020187200605869295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,1,0.019764800369739533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,1,0.020054399967193604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,1,0.01982239931821823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,1,0.02001280039548874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,3,0.015692800283432007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,3,0.014183999598026275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,3,0.014300799369812012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,3,0.0142752006649971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,3,0.013896000385284425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,3,0.014262400567531586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,3,0.0142752006649971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,3,0.021622399985790252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,3,0.020260800421237946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,3,0.019734400510787963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,3,0.01998240053653717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,3,0.01998240053653717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,3,0.019892799854278564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,3,0.01982560008764267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,7,0.015435199439525604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,7,0.01422560065984726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,7,0.014207999408245086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,7,0.01422560065984726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,7,0.013920000195503235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,1,0.013967999815940857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,7,0.014033600687980652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,7,0.014283199608325959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,7,0.020286400616168977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,1,0.013847999274730682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,7,0.019819200038909912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,7,0.019728000462055206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,7,0.01966879963874817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,7,0.01974720060825348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,15,0.014683200418949128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,15,0.014321599900722504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,15,0.01419519931077957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,15,0.014238399267196656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,15,0.014219200611114502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,15,0.014457599818706512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,15,0.022096000611782074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,15,0.020681600272655486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,15,0.020623999834060668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,15,0.020750400424003602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,15,0.020310400426387785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,15,0.02091200053691864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,15,0.020819200575351714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,31,0.016279999911785126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,31,0.014681600034236908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,31,0.014604799449443817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,2,65535,4.369811248779297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,31,0.014368000626564025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,31,0.014467200636863709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,32,65535,4.385507202148437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,31,0.014139199256896972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,31,0.022784000635147093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,31,0.023839999735355378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,31,0.022468799352645875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,7,0.019782400131225585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,31,0.022392000257968902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,31,0.02245119959115982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,63,0.016846400499343873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,15,0.015548799932003022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,63,0.015617600083351136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,63,0.015265600383281707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,63,0.014727999269962311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,63,0.014691199362277984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,63,0.015110400319099427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,63,0.02709600031375885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,63,0.02621600031852722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,63,0.02595199942588806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,63,0.025758400559425354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,63,0.02561439871788025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,63,0.025723201036453248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,63,0.025649601221084596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,127,0.01934240013360977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,127,0.016710400581359863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,127,0.01672160029411316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,127,0.016832000017166136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,127,0.01635040044784546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,31,0.014500799775123595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,127,0.016910399496555328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,127,0.016808000206947327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,127,0.034599998593330385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,127,0.03363359868526459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,127,0.03333599865436554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,127,0.033099201321601865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,127,0.0329008013010025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,127,0.03317280113697052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,1,0.020015999674797058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,127,0.03325279951095581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,255,0.022071999311447144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,255,0.020294399559497835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,255,0.019896000623703003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,255,0.019705599546432494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,255,0.0194815993309021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,255,0.02004159986972809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,63,0.015267199277877808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,255,0.05478559732437134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,255,0.051652801036834714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,255,0.050012797117233276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,255,0.04944320023059845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,255,0.049404799938201904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,255,0.04920479953289032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,255,0.04910080134868622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,511,0.0461903989315033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,511,0.037136000394821164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,511,0.032716798782348636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,511,0.026868799328804018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,511,0.026423999667167665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,511,0.02688480019569397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,511,0.028123199939727783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,511,0.10518079996109009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,511,0.09717440009117126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,511,0.09237279891967773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,511,0.0934831976890564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,511,0.09457280039787293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,511,0.09439359903335572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,1023,0.06192960143089295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,1023,0.05240960121154785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,1023,0.047926399111747744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,7,0.021353599429130555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,1023,0.04583680033683777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,31,0.022435200214385987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,1023,0.0476063996553421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,1023,0.046744000911712644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,1023,0.17802720069885253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,255,0.0200095996260643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,1023,0.16996320486068725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,1023,0.16674720048904418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,1023,0.16563520431518555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,1023,0.16725759506225585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,1023,0.1664944052696228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,2047,0.0871936023235321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,2047,0.0731119990348816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,2047,0.07070720195770264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,2047,0.07155359983444214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,2047,0.0722607970237732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,511,0.09353920221328735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,2047,0.07248799800872803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,2047,0.3175472021102905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,2047,0.3103728055953979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,2047,0.30678720474243165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,2047,0.302892804145813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,1,0.02131360024213791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,2047,0.3063999891281128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,2047,0.3051055908203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,1023,0.0463919997215271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,31,0.02234400063753128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,4095,0.1372655987739563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,2047,0.30650880336761477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,4095,0.12254559993743896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,4095,0.12076640129089355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,4095,0.12280960083007812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,4095,0.12474080324172973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,4095,0.12493280172348023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,1023,0.16514719724655152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,4095,0.5888832092285157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,4095,0.5829296112060547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,4095,0.5790671825408935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,2047,0.07742559909820557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,4095,0.576473617553711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,4095,0.5802288055419922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,4095,0.5824399948120117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,8191,0.23816161155700682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,8191,0.21949119567871095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,4095,0.5801839828491211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,8191,0.22951040267944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,8191,0.22912321090698243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,8191,1.1371088027954102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,8191,1.1255599975585937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,8191,1.1262096405029296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,8191,1.1227055549621583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,4095,0.12786079645156861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,16383,0.43773598670959474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,16383,0.4263472080230713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,8191,1.127403163909912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,8191,1.128439998626709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,8191,1.1338640213012696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,8191,0.22301440238952636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,8191,0.22604000568389893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,16383,0.42520480155944823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,16383,0.4272304058074951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,16383,2.229747200012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,16383,2.203539276123047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,8191,0.2273535966873169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,16383,0.428004789352417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,16383,2.209467124938965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,16383,2.2237407684326174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,16383,2.2217439651489257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,16383,2.2231088638305665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,1,32767,0.8341407775878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,16383,0.42238879203796387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,2,32767,0.8234895706176758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,4,32767,0.8176480293273926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,32767,0.8132927894592286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,16,32767,0.8193504333496093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,32,32767,0.8244208335876465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,64,1,64,32767,0.8239472389221192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,16383,2.2171808242797852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,8,16383,0.4178368091583252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,1,0.017923200130462648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,1,0.016067199409008026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,1,0.015617600083351136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,1,0.015540799498558045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,1,0.015505599975585937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,1,0.015548799932003022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,1,0.01555519998073578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,2,32767,4.405436706542969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,4,32767,4.403424072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,1,0.02176479995250702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,1,0.021931199729442595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,1,0.02138399928808212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,8,32767,4.392948913574219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,1,0.021881599724292756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,1,0.02175839990377426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,3,0.017627200484275816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,1,0.02131199985742569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,3,0.015919999778270723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,3,0.015609599649906158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,3,0.015297600626945495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,3,0.01560640037059784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,3,0.02407200038433075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,32,32767,4.406265640258789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,3,0.02226399928331375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,3,0.021252800524234772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,3,0.021908800303936004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,3,0.0216511994600296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,3,0.021694399416446686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,7,0.017796799540519714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,7,0.01639840006828308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,7,0.01579679995775223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,7,0.01565759927034378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,7,0.015369600057601929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,1,32767,4.401145553588867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,7,0.015731200575828552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,7,0.023849600553512575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,7,0.015777599811553956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,7,0.022606399655342103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,7,0.02189600020647049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,7,0.022224000096321105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,7,0.022126400470733644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,7,0.021984000504016877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,7,0.022145600616931917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,15,0.017667199671268462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,16,32767,4.4013214111328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,15,0.015667200088500977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,3,0.015432000160217285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,15,0.016011199355125426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,15,0.01610559970140457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,15,0.02651199996471405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,15,0.024396799504756927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,15,0.024568000435829164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,15,0.024475200474262236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,3,0.02141280025243759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,15,0.024403199553489685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,15,0.023924799263477327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,31,0.017936000227928163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,31,0.016072000563144683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,31,0.016169600188732147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,64,1,64,32767,4.416523361206055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,31,0.01599839925765991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,31,0.015643200278282164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,31,0.0159168004989624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,31,0.01600960046052933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,31,0.026921600103378296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,31,0.029099199175834655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,31,0.026756799221038817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,31,0.026764801144599913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,31,0.0268095999956131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,31,0.02643359899520874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,31,0.026785600185394286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,63,0.01918399930000305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,63,0.016972799599170686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,63,0.016420799493789672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,63,0.016655999422073364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,15,0.015958400070667268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,3,0.015659199655056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,63,0.016739200055599212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,63,0.03665440082550049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,63,0.0335536003112793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,63,0.03344799876213074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,63,0.03338559865951538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,63,0.03317759931087494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,63,0.03291679918766022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,63,0.033460798859596255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,127,0.021611200273036958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,127,0.01897760033607483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,127,0.019096000492572783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,127,0.018654400110244752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,127,0.01908160001039505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,15,0.024408000707626342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,127,0.01839520037174225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,127,0.019036799669265747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,127,0.05772799849510193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,127,0.05255200266838074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,127,0.05056639909744263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,127,0.04909439980983734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,127,0.048046401143074034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,127,0.04836480021476745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,1,0.024087999761104584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,255,0.034799998998641966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,255,0.03142560124397278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,255,0.02576479911804199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,63,0.01666879951953888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,255,0.02338239997625351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,63,0.016067199409008026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,255,0.09351680278778077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,15,0.015537600219249725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,15,0.01570879966020584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,255,0.09057760238647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,255,0.08899999856948852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,255,0.08787840008735656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,255,0.08825439810752869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,255,0.08789439797401429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,255,0.08797600269317626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,511,0.05282400250434875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,511,0.04800640046596527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,511,0.04606559872627258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,511,0.04458720088005066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,511,0.04384320080280304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,511,0.044200000166893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,511,0.045049598813056944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,511,0.17105599641799926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,511,0.16719199419021608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,511,0.16551680564880372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,511,0.16401920318603516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,511,0.16365760564804077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,511,0.16419199705123902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,1023,0.07842559814453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,1023,0.07356160283088684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,127,0.04968799948692322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,1023,0.07132480144500733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,1023,0.0693615972995758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,1023,0.06971520185470581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,1023,0.06890400052070618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,1023,0.3100352048873901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,1023,0.3076159954071045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,255,0.023768000304698944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,1023,0.3042304039001465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,1023,0.30460960865020753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,255,0.023287999629974365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,1023,0.3035936117172241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,1023,0.3022624015808105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,2047,0.12314560413360595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,1023,0.3044991970062256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,2047,0.12126879692077637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,2047,0.11955360174179078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,2047,0.12072160243988037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,2047,0.11942399740219116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,2047,0.11920000314712524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,2047,0.5863167762756347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,2047,0.5802512168884277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,511,0.16354559659957885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,1023,0.07015519738197326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,2047,0.5795392036437989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,2047,0.5761472225189209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,2047,0.5768159866333008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,4095,0.22749121189117433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,4095,0.22254879474639894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,2047,0.5766719818115235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,255,0.023904000222682954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,4095,0.21970560550689697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,4095,0.217958402633667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,4095,0.21818718910217286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,4095,0.21775999069213867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,4095,0.21728320121765138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,2047,0.1286512017250061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,4095,1.1291839599609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,4095,1.1274592399597168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,4095,1.1221360206604003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,4095,1.1222864151000977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,2047,0.5784448146820068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,4095,1.1212528228759766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,4095,1.120680046081543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,8191,0.4368015766143799
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,8191,0.41897759437561033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,8191,0.4217967987060547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,8191,0.4166255950927734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,8191,0.4158016204833984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,8191,0.4165088176727295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,8191,0.4166880130767822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,4095,1.1249391555786132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,8191,2.2076496124267577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,8191,2.222542381286621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,8191,2.2115535736083984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,8191,2.2104543685913085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,1,16383,0.8241935729980469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,2,16383,0.8168319702148438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,4,16383,0.815675163269043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,8,16383,0.8131584167480469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,16,16383,0.8141599655151367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,32,16383,0.8111040115356445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,128,1,64,16383,0.8113504409790039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,8191,2.216089630126953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,8191,2.204507255554199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,16383,4.372067260742187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,1,0.027979201078414916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,1,0.02317280024290085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,1,8191,2.2204431533813476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,1,0.022302399575710296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,2,16383,4.360688018798828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,4,16383,4.402436828613281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,1,0.02248000055551529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,1,0.022270399332046508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,1,0.03498719930648804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,1,0.022155199944972993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,1,0.029649600386619568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,1,0.028350400924682616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,16,16383,4.393510437011718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,32,16383,4.391556930541992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,3,0.02736319899559021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,3,0.023310400545597076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,3,0.022609600424766542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,3,0.022571200132369997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,3,0.021942399442195892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,3,0.022115199267864226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,3,0.022441600263118745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,3,0.03580319881439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,3,0.02993279993534088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,3,0.029447999596595765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,3,0.028566399216651918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,3,0.02889919877052307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,3,0.028921601176261903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,8,16383,4.375624084472657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,3,0.028667199611663818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,7,0.023694400489330292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,7,0.027985599637031556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,7,0.022987200319766997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,1,0.028839999437332155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,7,0.022097599506378175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,7,0.022280000150203705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,1,0.028142398595809935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,7,0.03943839967250824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,1,0.02839359939098358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,7,0.03136959969997406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,7,0.030875200033187868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,7,0.03075999915599823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,7,0.030622398853302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,15,0.028531199693679808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,1,0.028683200478553772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,15,0.023319999873638152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,15,0.02268960028886795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,15,0.02258400022983551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,15,0.022705599665641785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,15,0.022337600588798523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,15,0.022331200540065765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,15,0.04429920017719269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,15,0.03514240086078644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,15,0.033795198798179625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,15,0.03363359868526459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,15,0.03364959955215454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,15,0.03344320058822632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,15,0.03318719863891602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,31,0.030899199843406677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,31,0.023182399570941925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,31,0.022836799919605254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,128,1,64,16383,4.395987319946289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,31,0.022579200565814972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,31,0.02276960015296936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,31,0.02221599966287613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,31,0.022603200376033784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,31,0.042096000909805295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,31,0.03933440148830414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,31,0.041022399067878725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,31,0.03925119936466217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,31,0.039259201288223265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,31,0.038851198554039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,63,0.03783040046691895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,63,0.025279998779296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,63,0.02396000027656555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,63,0.023887999355793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,63,0.02354719936847687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,63,0.023787200450897217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,7,0.022595199942588805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,63,0.023796799778938293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,63,0.06926559805870056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,63,0.0618287980556488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,63,0.05589439868927002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,63,0.05399680137634277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,7,0.030559998750686646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,63,0.05340480208396912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,127,0.044198399782180785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,127,0.03905600011348724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,127,0.036248001456260684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,127,0.032915198802948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,127,0.029633599519729614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,127,0.029665601253509522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,127,0.029275199770927428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,127,0.10661120414733886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,127,0.10095039606094361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,127,0.09766560196876525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,127,0.0956719994544983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,127,0.09491040110588074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,31,0.052052801847457884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,127,0.09485440254211426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,127,0.09488319754600524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,255,0.05852800011634827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,255,0.0528544008731842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,1,0.022407999634742735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,255,0.049420800805091855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,255,0.048440000414848326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,255,0.048019200563430786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,255,0.04718720018863678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,255,0.17863359451293945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,255,0.17253439426422118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,7,0.022243200242519377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,63,0.057923197746276855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,255,0.17081279754638673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,255,0.16895680427551268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,7,0.03161279857158661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,63,0.0545087993144989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,511,0.08445119857788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,255,0.16903200149536132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,511,0.08029919862747192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,511,0.07740319967269897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,511,0.07607359886169433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,511,0.075900799036026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,511,0.32730081081390383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,511,0.31884639263153075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,511,0.3151263952255249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,511,0.31296160221099856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,511,0.3123408079147339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,511,0.3117392063140869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,1023,0.14311519861221314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,511,0.31209759712219237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,1023,0.13369280099868774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,1023,0.12932640314102173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,255,0.04732959866523743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,1023,0.1259376049041748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,1023,0.12547999620437622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,1023,0.12475680112838745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,1023,0.12443360090255737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,1023,0.600374412536621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,255,0.16844639778137208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,1023,0.590937614440918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,1023,0.5830656051635742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,511,0.07640960216522216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,2047,0.24112319946289062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,1023,0.5842959880828857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,1023,0.58296799659729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,2047,0.22783999443054198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,2047,0.2253551959991455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,2047,0.2291775941848755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,2047,0.22333118915557862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,2047,0.22376799583435059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,2047,1.146993637084961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,2047,1.1338720321655273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,2047,1.132265567779541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,1023,0.5870672225952148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,1023,0.5846496105194092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,2047,1.1317855834960937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,2047,0.23276801109313966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,2047,1.1233584403991699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,2047,1.126535987854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,2047,1.1282719612121581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,4095,0.440334415435791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,4095,0.4217072010040283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,4095,0.4198272228240967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,4095,0.4200128078460693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,4095,0.42049760818481446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,511,0.09408479928970337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,4095,2.219102478027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,4095,2.223846435546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,4095,2.2183839797973635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,4095,2.2106496810913088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,4095,0.42459521293640134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,4095,0.4300879955291748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,4095,2.2159008026123046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,1,8191,0.8790415763854981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,2,8191,0.8441856384277344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,4,8191,0.8192463874816894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,8,8191,0.8162879943847656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,4095,2.21517276763916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,255,0.16881439685821534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,16,8191,0.8138400077819824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,256,1,64,8191,0.8150848388671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,4095,2.2076160430908205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,32,8191,0.8319487571716309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,2,8191,4.390036773681641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,1,0.054830402135849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,4,8191,4.401355361938476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,1,0.03807039856910706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,1,0.04209280014038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,1,0.03707199990749359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,16,8191,4.382380676269531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,1,0.03707999885082245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,1,0.03716320097446442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,32,8191,4.4091136932373045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,1,0.05040799975395203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,1,0.06147999763488769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,1,0.04393920004367828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,1,0.044556799530982974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,1,0.04336479902267456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,3,0.05336800217628479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,3,0.0410863995552063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,3,0.03762080073356629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,3,0.037099200487136844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,3,0.03687039911746979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,3,0.03685919940471649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,3,0.03652159869670868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,3,0.06385120153427123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,3,0.05261279940605164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,3,0.04635840058326721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,3,0.045547199249267575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,3,0.045531201362609866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,3,0.04537599980831146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,1,8191,4.388927841186524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,3,0.04488799870014191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,256,1,64,8191,4.390604782104492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,7,0.04131200015544891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,7,0.055657601356506346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,7,0.037136000394821164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,7,0.036687999963760376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,7,0.03695839941501618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,7,0.06726880073547363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,1,0.03603839874267578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,7,0.056251198053359985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,7,0.04951040148735046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,7,0.04822399914264679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,7,0.04774079918861389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,1,0.043612799048423766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,7,0.04783200025558472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,1,0.04304479956626892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,15,0.05546560287475586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,15,0.042320001125335696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,15,0.0380511999130249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,15,0.037108799815177916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,15,0.03680639863014221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,15,0.036831998825073244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,15,0.07463200092315674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,15,0.0644208014011383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,15,0.05702880024909973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,15,0.05469440221786499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,15,0.05356159806251526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,15,0.05380319952964783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,15,0.053692799806594846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,31,0.045974400639533994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,31,0.038217601180076596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,31,0.03709119856357575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,31,0.03691200017929077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,31,0.03669120073318481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,31,0.03691039979457855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,31,0.09154559969902039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,31,0.08026559948921204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,31,0.07435680031776429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,8,8191,4.40337905883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,7,0.03833119869232178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,31,0.0706767976284027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,7,0.036643201112747194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,31,0.06754559874534607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,63,0.06313279867172242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,31,0.06951839923858642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,63,0.05163840055465698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,63,0.047307199239730834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,63,0.042828801274299624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,63,0.04452959895133972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,7,0.04736480116844177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,63,0.04147840142250061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,63,0.040643200278282166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,63,0.12682559490203857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,63,0.1167904019355774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,63,0.10760159492492676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,63,0.10658880472183227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,63,0.10504319667816162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,127,0.07670400142669678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,127,0.0658399999141693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,127,0.060305601358413695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,127,0.05768960118293762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,127,0.05628960132598877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,127,0.055641597509384154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,127,0.05489280223846436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,31,0.057014399766922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,127,0.19616800546646118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,127,0.18134239912033082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,127,0.177291202545166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,127,0.17676160335540772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,127,0.17594560384750366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,255,0.10294239521026612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,127,0.17594399452209472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,255,0.09136319756507874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,255,0.08630560040473938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,255,0.08434240221977234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,255,0.0831712007522583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,255,0.08271520137786866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,255,0.08033120036125183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,31,0.06709439754486084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,255,0.3349423885345459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,255,0.31831040382385256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,15,0.037441599369049075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,63,0.11134719848632812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,255,0.3143728017807007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,63,0.10550400018692016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,255,0.3145616054534912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,255,0.31348960399627684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,511,0.17244640588760377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,511,0.14625760316848754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,511,0.14101120233535766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,511,0.13893760442733766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,511,0.13723360300064086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,127,0.18614879846572877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,511,0.6298143863677979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,511,0.6124608039855957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,511,0.6032383918762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,511,0.5990752220153809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,511,0.5971312046051025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,511,0.5958608150482178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,255,0.3235759973526001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,511,0.5967487812042236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,1023,0.2531008005142212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,1023,0.24290080070495607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,1023,0.23658399581909179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,1023,0.23707199096679688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,511,0.15477919578552246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,1023,0.23405919075012208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,511,0.1373039960861206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,1023,1.1672767639160155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,1023,1.1595071792602538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,1023,1.1452015876770019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,1023,1.1408432006835938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,1023,0.2684704065322876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,1023,1.1357536315917969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,1023,1.1405152320861816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,2047,0.4480639934539795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,1023,0.23267199993133544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,2047,0.43277921676635744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,2047,0.4388576030731201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,2047,0.4310912132263184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,2047,0.43023681640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,255,0.3160831928253174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,2047,2.2528751373291014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,2047,0.4671199798583984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,1023,1.1348575592041015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,2047,2.227401542663574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,2047,0.4298863887786865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,2,4095,0.8435872077941895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,2047,2.2275568008422852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,4,4095,0.83363037109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,2047,2.225436782836914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,2047,2.2230688095092774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,8,4095,0.8264320373535157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,16,4095,0.8229503631591797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,2047,2.234440040588379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,32,4095,0.8222960472106934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,512,1,64,4095,0.8223615646362304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,1,4095,0.8646896362304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,2047,2.2429439544677736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,1,0.09408640265464782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,1,4095,4.437820816040039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,1,0.0744480013847351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,1,0.0613647997379303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,2,4095,4.432928085327148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,1,0.0667087972164154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,4,4095,4.388199996948242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,16,4095,4.371436691284179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,32,4095,4.422004699707031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,1,0.1057136058807373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,1,0.07816320061683654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,1,0.06998720169067382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,1,0.06914880275726318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,1,0.06889920234680176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,1,0.0690608024597168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,3,0.09112319946289063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,3,0.07408000230789184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,3,0.06485919952392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,3,0.06107199788093567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,3,0.06091039776802063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,512,1,64,4095,4.391494369506836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,3,0.060387200117111205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,3,0.08861119747161865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,3,0.10990240573883056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,3,0.08036159873008727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,3,0.07278879880905151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,3,0.07134400010108947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,3,0.07144960165023803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,7,0.09394559860229493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,7,0.07518720030784606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,1,0.06070719957351685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,7,0.06574400067329407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,7,0.06136000156402588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,7,0.06045759916305542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,8,4095,4.424059295654297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,7,0.060052800178527835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,7,0.09535040259361267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,7,0.11935679912567139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,1,0.08677279949188232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,7,0.08138399720191955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,7,0.08780480027198792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,7,0.07819039821624756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,7,0.07717919945716858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,7,0.07694720029830933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,15,0.09640640020370483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,15,0.0757856011390686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,15,0.06752640008926392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,15,0.0610480010509491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,15,0.06085600256919861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,3,0.060468798875808714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,15,0.06087679862976074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,15,0.13219840526580812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,15,0.11244800090789794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,15,0.10242240428924561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,15,0.09779679775238037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,15,0.09651039838790894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,3,0.07323679924011231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,15,0.09387840032577514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,15,0.09197760224342347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,31,0.07831839919090271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,31,0.06936640143394471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,31,0.06439999938011169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,31,0.06178240180015564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,31,0.06073279976844788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,31,0.060433602333068846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,31,0.16462719440460205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,1,0.06041600108146668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,31,0.13465119600296022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,7,0.060247999429702756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,31,0.12726880311965943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,31,0.12570079565048217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,63,0.10856479406356812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,63,0.08592799901962281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,63,0.07721120119094849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,63,0.0737824022769928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,63,0.07206559777259827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,15,0.06024320125579834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,63,0.07083839774131775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,63,0.0709824025630951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,63,0.22743360996246337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,63,0.20756640434265136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,63,0.19753600358963014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,63,0.19457919597625734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,31,0.09796800017356873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,63,0.1919167995452881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,63,0.19112000465393067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,127,0.11216000318527222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,1,0.06040319800376892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,127,0.10004479885101318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,31,0.1296895980834961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,127,0.09111359715461731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,127,0.0903216004371643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,127,0.3653631925582886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,127,0.34451680183410643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,127,0.3324431896209717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,127,0.32855041027069093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,127,0.32340800762176514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,127,0.3249903917312622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,127,0.32196640968322754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,63,0.19164960384368895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,255,0.15986720323562623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,31,0.14417279958724977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,255,0.14269280433654785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,127,0.0950543999671936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,255,0.14045120477676393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,255,0.1394384026527405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,255,0.6358176231384277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,255,0.6128880023956299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,255,0.604150390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,255,0.5986112117767334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,255,0.5955743789672852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,255,0.18392479419708252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,255,0.5946559906005859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,255,0.14829440116882325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,511,0.3170207977294922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,255,0.591267204284668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,511,0.26283199787139894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,127,0.09277439713478089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,511,0.2549407958984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,511,0.2513200044631958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,511,0.2519263982772827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,511,1.2179183959960938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,511,1.1835568428039551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,511,1.165998363494873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,127,0.13285599946975707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,511,1.161348819732666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,511,0.2820784091949463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,511,1.1544416427612305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,31,0.12628159523010254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,511,0.2522160053253174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,1023,0.4625855922698975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,1023,0.5176752090454102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,1023,0.45122561454772947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,1023,0.44785118103027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,511,1.1579839706420898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,1023,0.4455904006958008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,1023,0.4425151824951172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,255,0.14390720129013063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,511,1.157089614868164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,1023,2.308468818664551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,1023,2.267767906188965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,1023,2.2655696868896484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,1023,2.2516271591186525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,1023,0.4816768169403076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,2,2047,0.8847807884216309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,4,2047,0.8581839561462402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,1023,2.241035270690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,1023,2.252739143371582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,1023,2.2295631408691405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,32,2047,0.8382016181945801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,16,2047,0.8399184226989747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,1,1024,1,64,2047,0.8361871719360352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,1,2047,0.9445263862609863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,8,2047,0.8465328216552734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,1,0.013368000090122224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,1,0.012916800379753113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,1,0.012454400211572647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,1,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,1,0.012100800126791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,1,0.012008000165224075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,1,0.01210239976644516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,1,0.01890240013599396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,1,0.01855680048465729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,1,0.018272000551223754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,1,2047,4.502438354492187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,2,2047,4.444131088256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,1,0.017876799404621124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,1,0.017953599989414214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,1,0.017867200076580048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,8,2047,4.3832447052001955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,4,2047,4.447151947021484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,3,0.012985600531101227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,3,0.013390399515628815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,32,2047,4.381375885009765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,16,2047,4.424692916870117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,3,0.012185599654912949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,3,0.012167999893426895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,3,0.012185599654912949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,3,0.018768000602722167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,3,0.018299199640750885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,3,0.01799360066652298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,3,0.017985600233078002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,3,0.017907199263572694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,3,0.017921599745750427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,7,0.012796799838542938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,7,0.012489599734544754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,7,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,7,0.012076800316572189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,7,0.012142399698495865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,7,0.01912800073623657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,7,0.018743999302387238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,7,0.018161599338054658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,7,0.018038399517536163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,7,0.01786399930715561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,1,1024,1,64,2047,4.429179382324219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,7,0.01788160055875778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,15,0.013228799402713775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,1,0.017975999414920805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,15,0.012876799702644348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,15,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,15,0.012107200175523757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,15,0.012108799815177918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,15,0.012179200351238251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,15,0.019017599523067474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,15,0.018747200071811677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,15,0.018607999384403228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,15,0.01801439970731735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,3,0.012246400117874146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,15,0.01797440052032471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,15,0.017761600017547608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,3,0.012003199756145477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,31,0.012995199859142303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,31,0.013124799728393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,31,0.01255200058221817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,7,0.013337600231170654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,31,0.012347199767827988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,31,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,31,0.019145600497722626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,31,0.018775999546051025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,31,0.018223999440670012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,31,0.017987200617790224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,31,0.018083199858665466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,7,0.018059200048446654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,31,0.018025599420070648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,31,0.018137599527835845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,63,0.013067199289798737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,63,0.012625600397586822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,63,0.012307199835777282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,63,0.012211199849843979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,63,0.012222400307655335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,63,0.01231039986014366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,15,0.012247999757528305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,63,0.018662400543689728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,63,0.018305599689483643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,63,0.01818079948425293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,63,0.018003199994564057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,63,0.017870399355888366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,15,0.01798879951238632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,63,0.01797440052032471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,3,0.019289599359035493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,127,0.014779199659824372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,31,0.012212800234556198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,127,0.01483680009841919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,127,0.01390880048274994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,127,0.014113600552082061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,127,0.013864000141620637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,31,0.012265600264072418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,127,0.014108799397945404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,127,0.020611199736595153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,127,0.020508800446987153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,127,0.020083199441432952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,127,0.0197952002286911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,127,0.019860799610614776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,127,0.019808000326156615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,127,0.01956319957971573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,255,0.01804320067167282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,255,0.01785759925842285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,255,0.01716800034046173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,255,0.01716800034046173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,255,0.017155200242996216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,255,0.016894400119781494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,255,0.017153599858283998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,255,0.02374880015850067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,255,0.0234607994556427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,255,0.02316959947347641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,255,0.02300959974527359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,255,0.022771200537681578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,255,0.02281759977340698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,63,0.013302400708198547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,511,0.018833599984645844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,511,0.01778720021247864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,511,0.016956800222396852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,511,0.016457599401474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,511,0.01765120029449463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,511,0.017657600343227386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,511,0.024910399317741395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,511,0.025593599677085875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,511,0.023856000602245332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,511,0.023017600178718567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,511,0.022755199670791627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,511,0.023820799589157105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,511,0.023931199312210084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,1023,0.01988160014152527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,1023,0.01918559968471527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,1023,0.018145599961280824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,1023,0.016927999258041383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,1023,0.01785600036382675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,1023,0.017956799268722533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,1023,0.027500799298286437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,1023,0.025750398635864258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,1023,0.024568000435829164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,1023,0.0237184002995491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,127,0.014337599277496338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,1023,0.02332800030708313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,1023,0.024452799558639528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,1023,0.024502399563789367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,2047,0.019833600521087645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,2047,0.0217616006731987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,2047,0.018742400407791137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,2047,0.01855359971523285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,2047,0.017735999822616578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,2047,0.01817599982023239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,2047,0.03038400113582611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,255,0.02274879962205887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,2047,0.028431999683380126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,2047,0.027115198969841003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,2047,0.026347199082374574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,2047,0.02563999891281128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,2047,0.02645919919013977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,2047,0.02624320089817047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,4095,0.02415039986371994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,4095,0.02234880030155182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,4095,0.020814399421215057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,4095,0.019675199687480927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,4095,0.01989919990301132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,63,0.019121600687503813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,4095,0.02014880031347275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,4095,0.03344480097293854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,1023,0.017334400117397307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,4095,0.03440479934215546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,4095,0.03183839917182922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,4095,0.03160960078239441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,4095,0.03166080117225647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,8191,0.027881601452827455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,8191,0.025992000102996828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,7,0.012076800316572189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,8191,0.023537600040435792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,8191,0.02274399995803833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,8191,0.021731199324131013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,8191,0.023358400166034698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,8191,0.024083200097084045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,8191,0.04318560063838959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,8191,0.04481120109558105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,2047,0.018220800161361694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,8191,0.03935680091381073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,8191,0.03965280055999756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,16383,0.03237920105457306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,511,0.019406400620937347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,16383,0.03099679946899414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,16383,0.02741119861602783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,4095,0.019118399918079378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,16383,0.0263264000415802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,16383,0.025390401482582092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,16383,0.02608320116996765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,16383,0.026475200057029726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,16383,0.0679967999458313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,16383,0.06619679927825928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,16383,0.06106240153312683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,16383,0.05798720121383667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,4095,0.03052479922771454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,16383,0.0562063992023468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,32767,0.04372960031032562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,32767,0.03930400013923645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,32767,0.03524479866027832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,32767,0.032523199915885925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,32767,0.03073599934577942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,32767,0.03025279939174652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,32767,0.03040800094604492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,32767,0.10535839796066285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,8191,0.04036960005760193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,8191,0.04106720089912415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,32767,0.10108640193939208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,8191,0.041315200924873355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,32767,0.09932799935340882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,32767,0.09676480293273926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,32767,0.09727039933204651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,65535,0.060099202394485476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,32767,0.09794719815254212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,65535,0.05208160281181336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,65535,0.052963197231292725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,65535,0.047091200947761536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,4095,0.03127520084381104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,65535,0.049507200717926025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,65535,0.04793440103530884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,65535,0.17617599964141845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,65535,0.18061599731445313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,65535,0.1725600004196167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,65535,0.17120319604873657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,65535,0.1691872000694275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,65535,0.17057759761810304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,65535,0.17044479846954347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1,1,1,131071,0.08951359987258911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,131071,0.08750879764556885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1,1,4,131071,0.08324800133705139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1,1,8,131071,0.07748799920082092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1,1,16,131071,0.07701600193977357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1,1,32,131071,0.07822880148887634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1,1,1,131071,0.3221168041229248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1,1,2,131071,0.3187328100204468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1,1,4,131071,0.3136800050735474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1,1,8,131071,0.3130575895309448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,32767,0.09585440158843994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,131071,0.31216959953308104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1,1,32,131071,0.31224000453948975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,1,0.013444800674915314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,1,0.012838399410247803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,131071,0.31177759170532227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1,1,2,65535,0.05802559852600098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,1,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,1,0.012094400078058242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,1,0.01228640004992485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,1,0.01897120028734207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1,1,16,16383,0.05610560178756714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,1,0.017836800217628478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,1,0.017815999686717987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1,1,64,16383,0.05651519894599914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,1,0.017560000717639922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,3,0.013439999520778656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,3,0.012531200051307678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,3,0.01239520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,3,0.01225920021533966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,3,0.01218239963054657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,3,0.012110400199890136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,3,0.018833599984645844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,3,0.01833759993314743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1,1,64,131071,0.07555999755859374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,3,0.01791999936103821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,3,0.01775519996881485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,3,0.017657600343227386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,3,0.017694400250911714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,3,0.017712000012397765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,7,0.013462400436401368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,7,0.013027200102806091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,7,0.012559999525547028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,7,0.0123648002743721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,7,0.012292800098657608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,7,0.01218079999089241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,7,0.012166400253772736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,7,0.01876319944858551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,7,0.018369600176811218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,7,0.017921599745750427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,7,0.01765599995851517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,7,0.017772799730300902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,7,0.017716799676418305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,7,0.017720000445842744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,15,0.013376000523567199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,15,0.013022400438785553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,15,0.012683199346065521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,15,0.012484800070524216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,1,0.01241919994354248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,15,0.012319999933242797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,15,0.012177599966526032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,15,0.012249600142240524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,15,0.019073599576950075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,15,0.01844480037689209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,15,0.017852799594402315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,15,0.01788959950208664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,1,0.01835999935865402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,15,0.017819200456142426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,15,0.01780959963798523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,15,0.017548799514770508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,1,0.017452800273895265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,31,0.013041600584983826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,31,0.012455999851226807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,31,0.012167999893426895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,1,0.017611199617385866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,31,0.012134400010108948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,31,0.01210559979081154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,31,0.0190528005361557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,31,0.01844799965620041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,31,0.017921599745750427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,31,0.017849600315093993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,31,0.017851200699806214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,31,0.0176144003868103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,31,0.017759999632835387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,63,0.013355199992656708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,63,0.012993599474430084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,63,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,63,0.012289600074291229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,63,0.012249600142240524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,63,0.012139199674129486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,63,0.019072000682353974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,63,0.018412800133228303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,63,0.01817599982023239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,63,0.017955200374126436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,63,0.01794240027666092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,63,0.017852799594402315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,63,0.017892800271511078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,127,0.014758400619029999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,127,0.014283199608325959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,127,0.014035199582576752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,127,0.013929599523544311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,127,0.0139055997133255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,127,0.013910399377346038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,127,0.02077440023422241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,127,0.020329600572586058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,127,0.019892799854278564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,127,0.01966080069541931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,127,0.019649599492549897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,127,0.01943040043115616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,127,0.019521600008010863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,255,0.018222400546073915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,1,0.012100800126791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,255,0.01790879964828491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,255,0.017246399819850922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,255,0.01719360053539276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,255,0.016926400363445282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,255,0.017177599668502807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,31,0.013478399813175201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,255,0.016888000071048737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,255,0.023940800130367278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,255,0.02346719950437546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,255,0.022750400006771088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,255,0.02264160066843033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,255,0.022580799460411072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,255,0.02256480008363724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,511,0.019651199877262115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,511,0.018966400623321535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,511,0.018055999279022218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,511,0.017262400686740877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,63,0.0121568001806736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,511,0.017844800651073457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,511,0.01780800074338913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,511,0.025998398661613464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,511,0.025246399641036987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,511,0.02404319941997528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,511,0.023035199940204622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,511,0.02300640046596527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,127,0.015039999783039094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,511,0.02375999987125397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,511,0.024321599304676055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,1023,0.02019679993391037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,1023,0.017387199401855468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,1023,0.017015999555587767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,1023,0.017824000120162962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,1023,0.01810240000486374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,1023,0.029179200530052185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,1023,0.027932798862457274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,1023,0.026524800062179565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,1023,0.025563201308250426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,1023,0.025167998671531678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,1023,0.025879999995231627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,1023,0.02611039876937866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,2047,0.022120000422000886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,2047,0.021062399446964263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,2047,0.01921759992837906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,2047,0.018379199504852294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,2047,0.01810240000486374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,2047,0.018310399353504182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,2047,0.018380799889564516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,2047,0.03388960063457489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,31,0.012318400293588638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,2047,0.03177280128002167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,2047,0.03025600016117096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,2047,0.029488000273704528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,2047,0.029156801104545594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,2047,0.029241600632667543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,2047,0.02959200143814087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,4095,0.025526401400566102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,4095,0.02319840043783188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,4095,0.020817600190639496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,4095,0.02046239972114563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,4095,0.019654400646686554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,4095,0.02028159946203232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,4095,0.02064319998025894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,4095,0.04388799965381622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,4095,0.04136799871921539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,4095,0.037136000394821164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,1023,0.01929280012845993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,4095,0.03665600121021271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,4095,0.03733119964599609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,4095,0.037652799487113954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,8191,0.031031998991966247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,8191,0.028233599662780762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,8191,0.02388000041246414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,8191,0.022991999983787537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,8191,0.022111999988555908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,8191,0.025014400482177734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,8191,0.02553279995918274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,8191,0.06609759926795959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,8191,0.06424160003662109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,8191,0.05755680203437805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,8191,0.0556335985660553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,8191,0.05249599814414978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,8191,0.055848002433776855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,8191,0.055276799201965335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,16383,0.04237279891967773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,16383,0.039284801483154295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,16383,0.032558399438858035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,16383,0.030432000756263733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,16383,0.027659198641777037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,16383,0.02974880039691925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,16383,0.030372801423072814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,16383,0.10355039834976196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,16383,0.09938240051269531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,3,0.012947200238704682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,16383,0.09420160055160523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,16383,0.09494559764862061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,16383,0.09168800115585327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,16383,0.09634079933166503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,32767,0.05936639904975891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,32767,0.05529599785804749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,32767,0.049035200476646425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,32767,0.04902240037918091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,32767,0.045577600598335266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,32767,0.048932799696922304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,32767,0.048456001281738284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,4095,0.038011199235916136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,32767,0.17863520383834838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,32767,0.17378400564193724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,1023,0.01836320012807846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,32767,0.16940959692001342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,32767,0.16700960397720338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,32767,0.1681663990020752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,32767,0.16843039989471437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,65535,0.08733760118484497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,65535,0.08432000279426574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,65535,0.07887679934501649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,65535,0.07306240200996399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,65535,0.07481439709663391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,65535,0.07577440142631531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,65535,0.07285439968109131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,65535,0.3196016073226929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,65535,0.3150144100189209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,65535,0.3120687961578369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,65535,0.3089103937149048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,65535,0.3081056118011475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,2,1,1,131071,0.14531519412994384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,255,0.02279040068387985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,2,1,2,131071,0.13339359760284425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,2,1,4,131071,0.1300320029258728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,2,1,8,131071,0.127455997467041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,131071,0.12330080270767212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,16383,0.0972208023071289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,2,1,32,131071,0.1265936017036438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,2,1,64,131071,0.12647680044174195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,2,1,1,131071,0.5991263866424561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,2,1,2,131071,0.5895472049713135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,2,1,8,131071,0.5846079826354981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,32767,0.16946560144424438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,2,1,16,131071,0.5828512191772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,1,0.013447999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,131071,0.5837456226348877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,131071,0.5843455791473389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,1,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,1,0.012153600156307221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,1,0.012319999933242797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,1,0.012091200053691863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,1,0.019049599766731262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,1,0.018622399866580965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,1,0.01806560009717941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,1,0.017817600071430205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,1,0.017713600397109987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,1,0.017870399355888366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,1,0.017825600504875184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,3,0.013219200074672699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,2,1,64,65535,0.3103440046310425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,3,0.012801599502563477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,3,0.012464000284671784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,3,0.012120000272989272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,3,0.012227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,3,0.012223999947309494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,2,1,16,511,0.016972799599170686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,3,0.01900479942560196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,3,0.01875839978456497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,3,0.018216000497341157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,3,0.018049600720405578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,3,0.01786399930715561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,3,0.01767680048942566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,2,1,4,131071,0.585641622543335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,7,0.013361600041389466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,7,0.012852799892425538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,7,0.01218400001525879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,7,0.012135999649763108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,7,0.012142399698495865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,1,0.012534399330615998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,7,0.01908479928970337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,7,0.018886399269104005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,7,0.018401600420475006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,7,0.01812800019979477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,7,0.018001599609851836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,7,0.01805119961500168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,7,0.01818400025367737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,2,1,32,65535,0.3093247890472412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,15,0.013465599715709686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,15,0.01292479932308197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,15,0.012454400211572647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,15,0.012275200337171555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,3,0.012127999961376191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,15,0.01226079985499382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,15,0.012151999771595002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,15,0.0190528005361557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,15,0.01866080015897751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,15,0.018129600584506987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,15,0.018140800297260284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,15,0.017798399925231932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,15,0.01802079975605011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,15,0.01778080016374588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,31,0.013321599364280701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,31,0.012998400628566742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,31,0.012476799637079239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,31,0.012359999865293504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,31,0.012172800302505494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,31,0.01231359988451004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,31,0.012095999717712403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,31,0.01902880072593689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,31,0.018654400110244752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,31,0.018111999332904815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,31,0.018105599284172057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,31,0.017846399545669557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,31,0.018003199994564057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,31,0.01796640008687973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,7,0.012531200051307678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,63,0.013595199584960938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,63,0.013030399382114411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,63,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,63,0.012377600371837615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,63,0.012427199631929398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,63,0.01220960021018982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,63,0.01910399943590164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,63,0.018700799345970152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,63,0.018379199504852294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,63,0.01809599995613098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,7,0.012191999703645706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,63,0.017977599799633027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,63,0.018161599338054658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,1,0.01212640032172203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,127,0.014924800395965577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,127,0.014750400185585022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,127,0.014321599900722504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,127,0.014076800644397735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,127,0.014033600687980652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,127,0.013948799669742584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,127,0.020708799362182617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,127,0.020440000295639037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,127,0.020043200254440306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,127,0.019867199659347533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,127,0.01979839950799942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,127,0.01980320066213608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,127,0.019871999323368073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,255,0.018004800379276275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,255,0.01781280040740967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,255,0.017407999932765962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,255,0.01716800034046173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,255,0.01732800006866455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,255,0.017102399468421937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,255,0.016964800655841827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,255,0.024302400648593903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,255,0.023635199666023253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,255,0.02327840030193329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,255,0.023342399299144743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,255,0.0234047994017601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,255,0.023468799889087677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,511,0.020182399451732634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,511,0.019750399887561797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,3,0.01802240014076233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,511,0.01804800033569336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,511,0.017638400197029114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,511,0.01812160015106201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,511,0.018287999927997588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,511,0.028935998678207397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,63,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,511,0.028084799647331238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,511,0.026481598615646362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,511,0.025998398661613464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,511,0.025167998671531678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,511,0.026552000641822816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,511,0.02633439898490906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,1023,0.02229599952697754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,1023,0.019582399725914003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,63,0.01823360025882721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,1023,0.018415999412536622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,1023,0.017526400089263917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,127,0.01408960074186325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,1023,0.01826079934835434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,1023,0.01812479943037033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,1023,0.03377600014209747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,1023,0.03141280114650726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,1023,0.0295087993144989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,1023,0.028798401355743408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,1023,0.02875039875507355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,1023,0.02916319966316223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,2047,0.026369601488113403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,2047,0.022617599368095397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,2047,0.020372800529003143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,2047,0.01974239945411682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,2047,0.018632000684738158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,2047,0.0191103994846344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,2047,0.019329600036144257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,2047,0.043556800484657286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,255,0.02380480021238327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,2047,0.037401598691940305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,2047,0.03622399866580963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,2047,0.035945600271224974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,2047,0.03598560094833374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,2047,0.036471998691558837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,4095,0.031070399284362792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,4095,0.02574400007724762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,4095,0.021491199731826782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,4095,0.02075359970331192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,4095,0.022500799596309663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,4095,0.022771200537681578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,511,0.017059199512004852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,4095,0.06575679779052734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,4095,0.06101279854774475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,4095,0.0565887987613678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,4095,0.05374720096588135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,4095,0.051798397302627565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,4095,0.05314559936523437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,4095,0.05339840054512024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,8191,0.04325920045375824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,8191,0.038247999548912046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,8191,0.02872479856014252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,1023,0.01740960031747818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,8191,0.02963840067386627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,8191,0.029707199335098265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,1023,0.029075199365615846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,8191,0.10373120307922364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,8191,0.09714879989624023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,8191,0.09223840236663819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,8191,0.09277120232582092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,15,0.012556800246238708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,8191,0.09242079854011535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,8191,0.09627519845962525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,8191,0.09681119918823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,16383,0.059569597244262695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,16383,0.05405279994010925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,16383,0.049185600876808164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,16383,0.044891199469566344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,16383,0.045793598890304564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,16383,0.04935680031776428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,16383,0.04758239984512329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,16383,0.1784608006477356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,16383,0.17196799516677858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,16383,0.16755839586257934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,16383,0.16430239677429198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,16383,0.16565120220184326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,16383,0.16958400011062622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,32767,0.08743039965629577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,32767,0.08402400016784668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,32767,0.07763199806213379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,32767,0.07153279781341552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,32767,0.07326560020446778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,32767,0.07599999904632568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,32767,0.07313600182533264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,32767,0.3187376022338867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,32767,0.313321590423584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,8191,0.03144319951534271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,32767,0.30721120834350585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,8191,0.025377601385116577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,32767,0.30599520206451414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,32767,0.3092063903808594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,32767,0.30922880172729494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,65535,0.14453439712524413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,65535,0.13373759984970093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,65535,0.12298400402069092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,65535,0.12410720586776733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,65535,0.12373759746551513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,2047,0.040382400155067444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,65535,0.12556639909744263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,4095,0.023852799832820893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,65535,0.5893919944763184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,65535,0.5834047794342041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,16383,0.1697872042655945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,65535,0.5815936088562011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,65535,0.5811855792999268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,65535,0.5828447818756104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,4,1,1,131071,0.25427680015563964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,4,1,2,131071,0.23442718982696534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,131071,0.23023200035095215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,4,1,8,131071,0.22678239345550538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,4,1,32,131071,0.2234015941619873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,4,1,64,131071,0.2275696039199829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,4,1,4,65535,0.127947199344635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,131071,1.1451583862304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,4,1,2,131071,1.1371631622314453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,4,1,1,65535,0.5979887962341308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,131071,1.1313648223876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,4,1,8,131071,1.1213135719299316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,1,0.014595200121402741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,4,1,16,131071,1.1247967720031737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,1,0.012963199615478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,4,1,32,131071,1.1303584098815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,4,1,4,32767,0.30950078964233396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,1,0.012377600371837615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,1,0.012436799705028534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,1,0.01231520026922226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,4,1,16,131071,0.2246272087097168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,1,0.019200000166893005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,1,0.01852159947156906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,1,0.01855199933052063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,1,0.018268799781799315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,1,0.018279999494552612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,1,0.018172800540924072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,3,0.01329279989004135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,3,0.012555199861526489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,3,0.01263200044631958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,3,0.01228799968957901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,3,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,3,0.01228479966521263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,3,0.019844800233840942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,3,0.01863359957933426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,3,0.01834080070257187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,3,0.018297599256038667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,3,0.018347199261188506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,3,0.018372799456119537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,7,0.014502400159835815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,7,0.01326880007982254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,7,0.012859199941158295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,7,0.012638400495052337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,7,0.012430399656295776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,7,0.012455999851226807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,7,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,7,0.019841599464416503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,7,0.01919520050287247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,7,0.018563200533390046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,7,0.018361599743366243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,7,0.01830720007419586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,65535,0.5842048168182373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,7,0.018427200615406036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,1,0.012139199674129486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,15,0.013716800510883332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,7,0.018134400248527527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,1,0.019446399807929993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,15,0.012852799892425538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,15,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,15,0.01268640011548996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,15,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,15,0.019758400321006776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,15,0.018915200233459474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,15,0.018702399730682374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,3,0.014235199987888336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,15,0.01839679926633835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,15,0.018382400274276733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,15,0.01836480051279068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,31,0.014555199444293976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,31,0.013321599364280701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,31,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,31,0.012647999823093415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,31,0.012460800260305405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,31,0.012593600153923034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,31,0.012644800543785095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,31,0.019976000487804412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,31,0.018934400379657747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,31,0.01865600049495697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,31,0.01852000057697296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,31,0.018534399569034576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,1,0.01242239996790886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,31,0.018457600474357606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,4,1,64,131071,1.130355167388916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,15,0.012444800138473511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,63,0.013214400410652161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,63,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,63,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,63,0.012464000284671784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,15,0.01823839992284775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,63,0.012593600153923034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,63,0.01950719952583313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,63,0.019419200718402863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,63,0.01879359930753708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,63,0.018769599497318268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,63,0.018537600338459016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,63,0.018680000305175783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,63,0.01831520050764084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,127,0.015248000621795654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,127,0.01480959951877594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,31,0.01902720034122467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,127,0.014180800318717957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,127,0.014190399646759033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,127,0.014164799451828003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,127,0.014319999516010285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,127,0.021753600239753722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,127,0.02117439955472946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,127,0.02054399996995926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,127,0.02064799964427948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,127,0.020345599949359895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,127,0.020686399936676026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,127,0.020449599623680113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,255,0.01839520037174225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,255,0.017948800325393678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,255,0.017283199727535246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,255,0.01719360053539276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,255,0.017151999473571777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,255,0.01720159947872162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,255,0.026606398820877075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,255,0.026027199625968934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,255,0.025603199005126955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,255,0.025308799743652344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,255,0.025193598866462708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,255,0.02518239915370941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,15,0.013091200590133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,511,0.02218720018863678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,511,0.02003680020570755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,511,0.01820479929447174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,511,0.017454400658607483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,511,0.017203199863433837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,511,0.018060800433158875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,3,0.01895360052585602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,511,0.03416160047054291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,127,0.014612799882888794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,511,0.03127520084381104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,511,0.030086401104927062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,511,0.029020801186561584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,511,0.028942400217056276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,511,0.029628801345825195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,511,0.03019680082798004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,1023,0.02510879933834076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,1023,0.022390399873256684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,1023,0.019920000433921815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,1023,0.018907199800014495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,1023,0.018199999630451203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,1023,0.01927199959754944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,255,0.017696000635623932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,1023,0.04603999853134155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,1023,0.04058400094509125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,1023,0.037038400769233704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,1023,0.03612639904022217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,1023,0.03592639863491058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,1023,0.036355200409889224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,1023,0.03665919899940491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,2047,0.032076799869537355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,2047,0.022593599557876588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,2047,0.020880000293254854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,2047,0.020398400723934174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,2047,0.02095839977264404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,2047,0.021524800360202788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,2047,0.06670079827308655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,2047,0.06173920035362244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,2047,0.055662399530410765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,2047,0.052585601806640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,63,0.01363680064678192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,2047,0.05193600058555603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,4095,0.04243359863758087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,63,0.012406399846076966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,4095,0.03513120114803314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,4095,0.03278239965438843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,4095,0.028339201211929323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,4095,0.025089600682258607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,4095,0.026571199297904968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,511,0.018486399948596955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,4095,0.10456960201263428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,4095,0.09625759720802307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,4095,0.09204000234603882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,4095,0.09106720089912415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,4095,0.09197440147399902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,4095,0.09219679832458497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,4095,0.09438560009002686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,8191,0.05845919847488403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,8191,0.05173599720001221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,1023,0.019889600574970245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,8191,0.04568000137805939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,8191,0.04627839922904968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,8191,0.04208320081233978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,8191,0.04933440089225769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,8191,0.04568159878253937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,8191,0.17876479625701905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,8191,0.17192959785461426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,8191,0.16591839790344237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,2047,0.05172160267829895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,8191,0.1689039945602417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,8191,0.16952320337295532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,16383,0.0858847975730896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,16383,0.07952640056610108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,16383,0.07320799827575683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,16383,0.06963359713554382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,16383,0.06693919897079467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,16383,0.07213919758796691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,16383,0.07229920029640198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,2047,0.027214398980140685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,16383,0.31736319065093993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,16383,0.3134768009185791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,8191,0.16540160179138183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,2047,0.05140960216522217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,16383,0.3050751924514771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,16383,0.30960960388183595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,16383,0.30955359935760496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,32767,0.13962240219116212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,4095,0.02685439884662628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,32767,0.12409119606018067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,32767,0.12487679719924927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,32767,0.12014720439910889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,32767,0.1258512020111084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,32767,0.12387839555740357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,32767,0.597111988067627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,32767,0.5872255802154541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,32767,0.5834767818450928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,32767,0.5791408061981201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,16383,0.30820159912109374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,32767,0.5817008018493652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,16383,0.3064255952835083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,65535,0.24886879920959473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,32767,0.5833327770233154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,65535,0.22875521183013917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,65535,0.2256767988204956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,32767,0.1303056001663208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,65535,0.22460479736328126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,65535,0.22720320224761964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,65535,0.22583360671997071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,65535,1.1358752250671387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,65535,1.1303728103637696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,65535,1.1249855995178222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,32767,0.5833183765411377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,255,0.0252703994512558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,65535,0.23489758968353272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,65535,1.1270895957946778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,65535,1.1297823905944824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,8,1,2,131071,0.43662238121032715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,65535,1.1278528213500976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,8,1,4,131071,0.43181281089782714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,8,1,8,131071,0.4295616149902344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,8,1,16,131071,0.4276576042175293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,8,1,32,131071,0.43055200576782227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,8,1,64,131071,0.43121919631958006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,65535,1.1469455718994142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,8191,0.16567840576171874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,8,1,1,131071,2.242724800109863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,8,1,2,131071,2.227948760986328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,8,1,4,131071,2.2161855697631836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,8,1,8,131071,2.2162544250488283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,1,0.014878399670124054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,1,0.014043200016021728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,1,0.012825599312782288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,8,1,16,131071,2.214892768859863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,1,0.012481600046157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,1,0.012435200065374375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,1,0.012481600046157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,1,0.018958400189876556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,1,0.020851199328899384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,1,0.018228800594806673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,1,0.018729600310325622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,1,0.01825920045375824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,3,0.015177600085735321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,3,0.013102400302886962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,3,0.012936000525951386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,3,0.012425599992275238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,3,0.012539200484752655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,3,0.012251199781894683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,3,0.012412799894809723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,3,0.02035360038280487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,8,1,32,131071,2.226553535461426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,3,0.01842080056667328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,3,0.018142400681972502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,3,0.018611200153827667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,3,0.018035200238227845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,3,0.01841920018196106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,7,0.013542400300502777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,7,0.012774400413036346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,8,1,64,131071,2.2334720611572267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,7,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,7,0.012824000418186187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,7,0.012783999741077422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,7,0.012644800543785095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,7,0.020664000511169435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,7,0.019947199523448943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,7,0.01868640035390854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,7,0.018745599687099455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,8,1,1,131071,0.46713762283325194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,7,0.018464000523090364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,7,0.01841440051794052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,1,0.01857440024614334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,1,0.01807039976119995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,15,0.013502399623394012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,15,0.012753599882125854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,15,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,15,0.012444800138473511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,15,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,15,0.012611199915409089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,15,0.020553599298000335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,15,0.019299200177192687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,15,0.018676799535751343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,15,0.018457600474357606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,15,0.01814880073070526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,3,0.018807999789714813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,15,0.01825760006904602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,31,0.014958399534225463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,31,0.013286399841308593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,31,0.01295199990272522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,31,0.01271039992570877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,7,0.014774399995803832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,31,0.012731200456619263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,31,0.01281599998474121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,31,0.021116800606250763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,31,0.02029760032892227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,31,0.0189983993768692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,31,0.018745599687099455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,31,0.018564799427986146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,31,0.01857600063085556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,31,0.018681600689888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,63,0.0151296004652977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,63,0.015036800503730774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,63,0.013062399625778199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,63,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,63,0.012667199969291687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,63,0.012720000743865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,63,0.020820799469947814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,63,0.02093919962644577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,63,0.019075199961662292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,63,0.0195360004901886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,7,0.018488000333309173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,63,0.01934559941291809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,63,0.019465599954128266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,1,0.0125231996178627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,63,0.01903039962053299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,127,0.015476800501346588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,15,0.015041600167751312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,127,0.015084800124168397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,127,0.01462399959564209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,127,0.014611199498176575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,127,0.01448799967765808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,127,0.024825599789619446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,127,0.023425599932670592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,127,0.022779199481010436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,127,0.022623999416828154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,127,0.02228800058364868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,127,0.02250719964504242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,127,0.022489599883556366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,255,0.019937600195407867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,15,0.018303999304771425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,255,0.018595199286937713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,255,0.01793439984321594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,255,0.01775840073823929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,255,0.017521600425243377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,255,0.018028800189495087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,255,0.030449599027633667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,255,0.029348799586296083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,255,0.02884480059146881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,255,0.028439998626708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,255,0.028355199098587035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,255,0.028279998898506166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,255,0.028203201293945313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,511,0.027753600478172304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,511,0.022785599529743194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,511,0.01977760046720505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,511,0.019091199338436126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,511,0.01812800019979477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,511,0.019472000002861024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,511,0.019729599356651306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,63,0.013257600367069244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,511,0.04641439914703369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,511,0.041228801012039185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,511,0.037064000964164734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,511,0.035641598701477054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,511,0.03621439933776856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,1023,0.035631999373435974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,1023,0.02717759907245636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,1023,0.022152000665664674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,1023,0.020982399582862854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,1023,0.02016319930553436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,1023,0.020908799767494202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,1023,0.021249599754810333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,127,0.01675039976835251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,1023,0.06916800141334534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,127,0.014975999295711518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,1023,0.0616815984249115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,1023,0.05612639784812927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,1023,0.053118401765823366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,1023,0.05128480195999145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,1023,0.05151519775390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,1023,0.05116959810256958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,2047,0.04424639940261841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,2047,0.037625598907470706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,2047,0.028207999467849732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,2047,0.024977600574493407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,2047,0.02552480101585388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,2047,0.025201600790023804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,2047,0.10679199695587158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,2047,0.09819200038909912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,31,0.012587200105190276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,2047,0.09346719980239868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,2047,0.0923695981502533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,2047,0.09196320176124573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,2047,0.09300640225410461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,2047,0.09236000180244446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,4095,0.05552160143852234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,4095,0.06035040020942688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,4095,0.04642559885978699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,4095,0.0417136013507843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,4095,0.046140798926353456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,4095,0.04409280121326446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,4095,0.17996000051498412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,4095,0.17185280323028565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,511,0.03566080033779144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,511,0.036550399661064145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,4095,0.16475679874420165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,4095,0.16354880332946778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,4095,0.16511039733886718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,8191,0.08792319893836975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,8191,0.08026559948921204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,8191,0.07094879746437073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,8191,0.06885920166969299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,2047,0.03249920010566711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,8191,0.07409600019454957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,255,0.01762080043554306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,8191,0.32015841007232665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,8191,0.31058559417724607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,4095,0.04757600128650665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,8191,0.30588641166687014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,8191,0.3041840076446533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,8191,0.30273120403289794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,8191,0.3082815885543823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,8191,0.3086735963821411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,16383,0.14257440567016602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,16383,0.12344319820404052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,16383,0.129966402053833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,16383,0.12036479711532592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,16383,0.12456320524215699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,4095,0.16664960384368896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,8191,0.07103199958801269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,16383,0.597756814956665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,16383,0.5841072082519532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,16383,0.5816336154937745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,8191,0.07307360172271729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,16383,0.5767199993133545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,16383,0.576862382888794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,16383,0.5807775974273681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,32767,0.25050399303436277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,32767,0.23125119209289552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,32767,0.2243743896484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,32767,0.22155520915985108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,32767,0.218558406829834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,32767,0.2227247953414917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,4095,0.164520001411438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,16383,0.11689280271530152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,32767,1.1304335594177246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,32767,1.126195240020752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,32767,1.1226160049438476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,32767,1.121572780609131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,32767,1.125707244873047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,16383,0.5840223789215088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,65535,0.4300816059112549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,32767,0.22236320972442628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,65535,0.42251200675964357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,65535,0.4199808120727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,65535,0.4181680202484131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,65535,0.4214975833892822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,65535,0.42151679992675783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,32767,1.148243236541748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,16383,0.12357120513916016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,32767,1.1232799530029296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,65535,2.264329528808594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,65535,2.2290496826171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,65535,2.2179168701171874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,65535,2.2152336120605467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,65535,0.4732480049133301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,65535,2.2112607955932617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,65535,2.2169519424438477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,16,1,1,131071,0.9339728355407715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,16,1,2,131071,0.8275919914245605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,16,1,4,131071,0.8197792053222657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,16,1,8,131071,0.817140769958496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,16,1,16,131071,0.8151696205139161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,16,1,32,131071,0.8190719604492187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,65535,2.21539363861084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,16,1,64,131071,0.8180208206176758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,16,1,2,131071,4.379998397827149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,16,1,1,131071,4.482764816284179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,16,1,4,131071,4.383343887329102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,16,1,8,131071,4.372276687622071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,1,0.015203200280666351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,1,0.013654400408267976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,16,1,16,131071,4.374528121948242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,1,0.013355199992656708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,1,0.013299199938774108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,1,0.01327199935913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,1,0.02147199958562851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,16,1,64,131071,4.385481643676758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,1,0.018905599415302277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,1,0.018838399648666383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,1,0.018931199610233308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,1,0.01892800033092499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,3,0.014902399480342865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,3,0.015787200629711153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,3,0.013660800457000733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,3,0.013376000523567199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,3,0.013246400654315949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,3,0.01324159950017929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,3,0.021555200219154358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,3,0.02075680047273636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,3,0.01942719966173172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,3,0.019313600659370423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,3,0.01915999948978424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,3,0.019038400053977965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,3,0.01900479942560196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,7,0.015532800555229187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,7,0.015041600167751312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,7,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,7,0.013372799754142762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,7,0.013255999982357025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,7,0.013167999684810638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,7,0.021452799439430237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,7,0.0202224001288414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,1,0.015878400206565856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,7,0.01907840073108673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,1,0.013312000036239623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,7,0.019206400215625762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,16,1,32,131071,4.3932750701904295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,7,0.019043199717998505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,1,0.020337599515914916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,1,0.01905599981546402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,15,0.01374559998512268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,15,0.013315199315547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,15,0.013364799320697784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,15,0.013612799346446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,15,0.021566399931907655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,3,0.013177600502967835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,15,0.020576000213623047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,15,0.01953279972076416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,15,0.019337600469589232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,15,0.019252799451351166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,15,0.019276799261569978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,15,0.019193600118160247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,31,0.01568000018596649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,31,0.015024000406265258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,31,0.014008000493049622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,31,0.013609600067138673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,31,0.013438400626182557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,31,0.013467200100421906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,31,0.013769599795341491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,31,0.0217056006193161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,31,0.019776000082492827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,31,0.01960960030555725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,31,0.019852800667285918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,31,0.019739200174808503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,31,0.01958719938993454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,63,0.015451200306415558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,63,0.015596799552440643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,63,0.014211200177669525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,7,0.013331200182437896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,63,0.01408960074186325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,63,0.014086399972438813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,63,0.013809600472450256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,63,0.013918399810791016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,63,0.02380799949169159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,63,0.023604799807071686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,63,0.022145600616931917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,63,0.02207999974489212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,63,0.021807999908924104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,15,0.014902399480342865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,63,0.021980799734592438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,15,0.015718400478363037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,127,0.01749439984560013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,127,0.017419199645519256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,15,0.013471999764442444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,127,0.015723200142383577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,127,0.015593600273132325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,127,0.0157600000500679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,127,0.028507199883461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,127,0.028171199560165405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,127,0.02647840082645416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,127,0.02632479965686798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,127,0.02610880136489868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,127,0.026233598589897156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,127,0.02611680030822754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,255,0.02104160040616989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,31,0.02101919949054718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,255,0.020137600600719452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,255,0.019204799830913544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,255,0.018796800076961516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,255,0.018671999871730804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,255,0.018702399730682374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,255,0.01889439970254898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,255,0.037241598963737486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,255,0.03681440055370331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,255,0.035492798686027525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,255,0.03531999886035919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,255,0.03476319909095764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,255,0.03510879874229431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,255,0.03489440083503723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,511,0.04000000059604645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,511,0.022675199806690215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,511,0.02935360074043274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,511,0.021996800601482392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,511,0.02067999988794327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,511,0.021568000316619873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,7,0.019281600415706635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,511,0.021855999529361726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,7,0.0191648006439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,511,0.07274399995803833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,511,0.06534720063209534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,511,0.05299680233001709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,511,0.051708799600601194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,511,0.05182880163192749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,511,0.05176640152931213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,127,0.015625600516796113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,1023,0.04851039946079254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,1023,0.04147680103778839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,1023,0.03365119993686676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,1023,0.03212960064411163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,1023,0.02489120066165924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,1023,0.024985599517822265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,1023,0.02446240037679672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,1023,0.10980639457702637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,1023,0.10109119415283203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,1023,0.09485120177268982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,1023,0.0900223970413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,1023,0.09242879748344421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,2047,0.0641424000263214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,2047,0.0572160005569458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,2047,0.0470335990190506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,2047,0.046900799870491026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,2047,0.04206399917602539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,2047,0.045347198843955994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,2047,0.042587199807167055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,2047,0.18341599702835082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,2047,0.17474240064620972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,2047,0.16568959951400758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,2047,0.1660416007041931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,2047,0.16474720239639282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,2047,0.16567519903182984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,2047,0.165009605884552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,4095,0.0933247983455658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,4095,0.08312479853630066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,4095,0.07313439846038819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,4095,0.07153919935226441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,4095,0.0671567976474762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,4095,0.07124480009078979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,511,0.057107198238372806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,63,0.021875199675559998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,127,0.01605280041694641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,4095,0.3241055965423584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,4095,0.3136176109313965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,4095,0.30763680934906007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,4095,0.30445280075073244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,1023,0.09111840128898621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,4095,0.30346240997314455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,4095,0.30580160617828367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,8191,0.13303359746932983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,8191,0.12473440170288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,8191,0.121288001537323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,8191,0.11906880140304565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,8191,0.12488479614257812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,8191,0.6004543781280518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,8191,0.5890192031860352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,8191,0.5804255962371826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,8191,0.5780352115631103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,8191,0.5780720233917236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,8191,0.5818975925445556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,4095,0.0693231999874115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,16383,0.23280479907989501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,16383,0.22462399005889894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,16383,0.21980960369110109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,16383,0.2185296058654785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,16383,0.22264480590820312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,1023,0.0928816020488739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,16383,0.22439520359039306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,8191,0.1239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,16383,1.1532095909118651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,16383,1.1372927665710448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,16383,1.1276991844177247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,16383,1.1249775886535645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,16383,0.25419681072235106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,8191,0.5818448066711426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,16383,1.1272336006164552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,16383,1.128048038482666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,16383,1.128059196472168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,32767,0.47272157669067383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,8191,0.14633439779281615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,32767,0.43250241279602053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,32767,0.4239823818206787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,32767,0.4190800189971924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,32767,0.4180287837982178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,32767,0.4228528022766113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,32767,0.42310237884521484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,32767,2.2557823181152346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,32767,2.225399971008301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,32767,2.2224895477294924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,32767,2.2131696701049806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,32767,2.2159568786621096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,4095,0.30652000904083254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,32,1,1,65535,0.9246463775634766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,32,1,2,65535,0.8300944328308105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,32,1,4,65535,0.820088005065918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,32,1,8,65535,0.8161711692810059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,32,1,32,65535,0.8184255599975586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,32767,2.2157520294189452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,32,1,64,65535,0.8184528350830078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,32767,2.2280160903930666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,32,1,16,65535,0.8123472213745118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,1,0.017267200350761413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,1,0.016169600188732147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,32,1,2,65535,4.393779373168945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,32,1,1,65535,4.4873504638671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,32,1,4,65535,4.396131134033203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,32,1,8,65535,4.398425674438476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,1,0.014228799939155578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,1,0.01390720009803772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,1,0.013899199664592743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,32,1,16,65535,4.391900634765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,1,0.021236799657344818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,32,1,32,65535,4.428692626953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,1,0.019728000462055206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,1,0.020326399803161622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,1,0.019753600656986236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,3,0.015801599621772765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,3,0.017467199265956877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,3,0.014480000734329224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,3,0.014235199987888336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,3,0.01438080072402954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,3,0.014105600118637086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,3,0.02344159930944443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,32,1,64,65535,4.406763076782227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,3,0.02172800004482269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,3,0.02004159986972809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,3,0.02029920071363449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,3,0.019894400238990785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,3,0.020291200280189513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,7,0.017582400143146514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,7,0.015615999698638916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,7,0.014284799993038177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,7,0.014239999651908874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,7,0.014008000493049622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,7,0.014416000247001648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,1,0.014313599467277527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,7,0.014227199554443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,7,0.02308479994535446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,7,0.021878400444984437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,7,0.02021760046482086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,7,0.02054399996995926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,7,0.020003199577331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,7,0.02017119973897934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,15,0.017446400225162507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,15,0.01605439931154251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,1,0.023844799399375914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,1,0.019968000054359437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,15,0.0146479994058609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,15,0.013995200395584106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,1,0.020329600572586058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,15,0.023443199694156647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,15,0.022168000042438508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,3,0.014177599549293518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,15,0.020734399557113647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,15,0.020235200226306916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,15,0.020822399854660036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,15,0.02043360024690628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,15,0.020656000077724456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,31,0.017083199322223665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,31,0.016182400286197662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,31,0.01446399986743927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,31,0.014097599685192109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,31,0.014220799505710601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,31,0.014480000734329224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,31,0.02553919851779938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,31,0.023876799643039702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,31,0.022596800327301027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,31,0.022303999960422517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,31,0.022308799624443054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,31,0.022375999391078948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,7,0.01988160014152527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,31,0.022422400116920472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,1,0.014227199554443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,63,0.017004799842834473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,63,0.01499519944190979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,63,0.014500799775123595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,15,0.014287999272346497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,63,0.014958399534225463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,15,0.01433439999818802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,63,0.014668799936771393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,63,0.0285616010427475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,63,0.027163198590278624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,63,0.02605760097503662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,63,0.025543999671936036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,63,0.02534559965133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,63,0.02561599910259247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,63,0.025379198789596557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,127,0.018972800672054292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,127,0.018164800107479097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,127,0.017188799381256104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,127,0.01690240055322647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,31,0.014847999811172486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,127,0.016369600594043732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,127,0.01684959977865219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,127,0.03777920007705689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,127,0.03452000021934509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,127,0.0337552011013031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,127,0.03309119939804077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,127,0.03281440138816834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,127,0.03275839984416962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,127,0.03295519948005676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,255,0.021238400042057036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,255,0.020259200036525725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,255,0.02025440037250519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,255,0.01959040015935898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,255,0.019729599356651306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,255,0.0198512002825737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,255,0.058855998516082766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,255,0.055032002925872806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,63,0.017374399304389953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,255,0.05110560059547424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,255,0.0499103993177414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,255,0.04903680086135864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,15,0.014635199308395385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,255,0.04973120093345642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,255,0.04852479994297028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,511,0.040833601355552675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,511,0.04591360092163086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,63,0.014584000408649444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,511,0.031108799576759338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,511,0.03192799985408783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,511,0.02746559977531433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,511,0.10462239980697632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,511,0.10479680299758912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,511,0.09527999758720399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,511,0.09371520280838012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,511,0.092222398519516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,511,0.09299359917640686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,127,0.0165583997964859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,511,0.09416800141334533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,1023,0.055297601222991946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,1023,0.061624002456665036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,1023,0.0519216001033783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,1023,0.044998401403427125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,1023,0.045572799444198606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,1023,0.04620159864425659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,1023,0.04434880018234253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,1023,0.17549439668655395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,1023,0.17820320129394532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,1023,0.16425280570983886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,1023,0.1658031940460205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,1023,0.16557279825210572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,1023,0.16741440296173096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,2047,0.08281760215759278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,2047,0.08711519837379456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,2047,0.07593759894371033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,2047,0.07279679775238038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,2047,0.06849920153617858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,2047,0.07088000178337098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,2047,0.06984959840774536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,511,0.03653280138969421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,511,0.026881599426269533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,2047,0.31579198837280276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,2047,0.3096287965774536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,2047,0.3057136058807373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,2047,0.30478720664978026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,2047,0.3062527894973755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,4095,0.13769760131835937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,4095,0.13729599714279175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,4095,0.12743200063705445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,4095,0.12229119539260865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,4095,0.1205456018447876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,255,0.0237184002995491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,4095,0.1228335976600647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,1023,0.16988639831542968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,4095,0.5928719997406006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,4095,0.590721607208252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,4095,0.5830912113189697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,4095,0.5772575855255127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,4095,0.5804831981658936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,4095,0.5822256088256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,2047,0.3162575960159302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,8191,0.24546239376068116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,8191,0.23866720199584962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,8191,0.22791039943695068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,2047,0.30431840419769285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,8191,0.22317759990692138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,8191,0.2199455976486206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,8191,0.2250080108642578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,8191,0.2288111925125122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,3,0.019787199795246124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,4095,0.12351360321044921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,8191,1.133668804168701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,8191,1.1306256294250487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,4095,0.5788959980010986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,8191,1.1241552352905273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,8191,1.1229583740234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,8191,1.128923225402832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,16383,0.4271967887878418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,16383,0.42144160270690917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,16383,0.4193280220031738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,8191,1.1284031867980957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,16383,0.42331838607788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,16383,0.42827520370483396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,8191,1.1466768264770508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,16383,2.2189903259277344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,16383,2.2030031204223635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,16383,2.2132192611694337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,16383,0.4667359828948975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,16383,2.2227344512939453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,64,1,1,32767,0.9304287910461426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,32767,0.835201644897461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,16383,2.2093759536743165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,16383,2.2073936462402344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,64,1,16,32767,0.8169631958007812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,64,1,8,32767,0.816868782043457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,64,1,32,32767,0.8208271980285644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,16383,2.245599937438965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,64,1,64,32767,0.8218576431274414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,64,1,2,16383,0.43727841377258303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,64,1,4,32767,0.8238479614257812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,1,0.028438401222229005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,1,0.017998400330543517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,1,0.01576479971408844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,1,0.015854400396347047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,1,0.01565439999103546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,1,0.01566080003976822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,1,0.01533920019865036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,64,1,1,32767,4.429159927368164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,1,0.0344976007938385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,1,0.021742400527000428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,1,0.023265600204467773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,1,0.021932800114154816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,64,1,4,32767,4.388232040405273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,64,1,2,32767,4.442820739746094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,64,1,16,32767,4.396233749389649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,1,0.021639999747276307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,3,0.02874079942703247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,64,1,32,32767,4.3498798370361325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,3,0.017556799948215483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,3,0.01595360040664673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,3,0.01564960032701492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,64,1,64,32767,4.4005615234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,3,0.0159168004989624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,3,0.02227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,3,0.036667200922966006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,3,0.02380480021238327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,3,0.021823999285697938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,3,0.021252800524234772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,3,0.02190079987049103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,7,0.027497598528862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,7,0.018880000710487364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,7,0.015907199680805208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,7,0.015830400586128234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,7,0.015545600652694702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,7,0.01600160002708435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,7,0.03678719997406006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,7,0.024430400133132933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,7,0.02260479927062988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,7,0.022401599586009978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,7,0.022152000665664674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,7,0.021884800493717195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,7,0.0224031999707222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,15,0.029259198904037477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,15,0.01807200014591217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,15,0.016096000373363496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,1,0.0211776003241539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,15,0.015668800473213194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,15,0.015911999344825744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,15,0.01584160029888153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,15,0.01576319932937622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,15,0.04132159948348999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,15,0.02613599896430969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,3,0.015860800445079804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,15,0.0243599995970726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,15,0.024140800535678863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,15,0.024246400594711302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,15,0.024609600007534028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,3,0.015398399531841278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,31,0.018564799427986146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,31,0.01602720022201538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,31,0.01615999937057495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,31,0.015956799685955047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,31,0.015987199544906617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,31,0.015585599839687348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,7,0.015835200250148774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,31,0.044588801264762876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,31,0.029275199770927428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,31,0.027184000611305235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,31,0.026897600293159483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,31,0.02701759934425354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,31,0.026679998636245726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,31,0.02654080092906952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,63,0.032620799541473386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,63,0.018785600364208222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,63,0.016996799409389494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,63,0.016790400445461272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,63,0.016820800304412842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,63,0.016761599481105803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,63,0.01634719967842102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,63,0.051532799005508424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,63,0.0379967987537384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,63,0.034179198741912845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,63,0.033451199531555176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,63,0.033486399054527285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,63,0.03312320113182068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,63,0.03298879861831665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,127,0.039710399508476255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,127,0.02182080000638962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,127,0.01879359930753708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,1,0.02162559926509857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,127,0.018934400379657747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,127,0.019631999731063842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,127,0.07199839949607849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,15,0.024700799584388734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,127,0.057036799192428586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,127,0.05267999768257141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,31,0.030417600274086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,127,0.0488783985376358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,127,0.049798399209976196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,127,0.04869279861450195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,255,0.04651199877262115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,255,0.03522560000419617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,255,0.02863999903202057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,3,0.021980799734592438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,255,0.023926399648189545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,255,0.023926399648189545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,255,0.02336000055074692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,255,0.10917439460754394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,255,0.09387999773025513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,255,0.08800960183143616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,255,0.08905119895935058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,255,0.08855680227279664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,255,0.08823999762535095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,255,0.08797600269317626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,511,0.07034080028533936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,511,0.05321440100669861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,511,0.04584159851074219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,511,0.04582720100879669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,511,0.04269599914550781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,511,0.04438399970531463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,511,0.19165600538253785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,64,1,8,32767,4.4103649139404295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,511,0.16854560375213623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,127,0.018668800592422485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,511,0.1652943968772888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,511,0.16444319486618042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,511,0.175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,127,0.04963839948177338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,511,0.16471999883651733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,511,0.16391520500183104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,1023,0.09946560263633727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,1023,0.07332000136375427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,1023,0.0674239993095398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,1023,0.07041119933128356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,255,0.02688960134983063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,1023,0.06746399998664857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,1023,0.33272318840026854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,1023,0.309881591796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,1023,0.3040751934051514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,1023,0.30244319438934325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,1023,0.30333919525146485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,511,0.04185760021209717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,2047,0.12853920459747314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,127,0.019444799423217772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,2047,0.12338240146636963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,2047,0.12100479602813721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,2047,0.12013280391693115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,2047,0.11923040151596069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,2047,0.6103184223175049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,2047,0.5838992118835449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,1023,0.07139840126037597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,1023,0.30576319694519044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,2047,0.5776768207550049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,2047,0.5792992115020752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,2047,0.5770544052124024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,1023,0.30291199684143066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,2047,0.5764287948608399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,4095,0.22730560302734376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,2047,0.5743919849395752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,2047,0.153603196144104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,4095,0.22205760478973388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,4095,0.2197648048400879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,4095,0.21869759559631347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,4095,0.21746559143066407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,2047,0.11914240121841431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,1023,0.07834240198135375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,4095,1.1308239936828612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,4095,1.162440013885498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,4095,1.1233856201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,4095,1.1240896224975585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,8191,0.48648638725280763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,8191,0.43727841377258303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,4095,1.1250304222106933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,4095,1.1225104331970215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,4095,1.1200511932373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,4095,0.26267199516296386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,8191,0.42202558517456057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,8191,0.41666240692138673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,8191,0.4168528079986572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,4095,0.21713919639587403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,8191,2.2628095626831053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,8191,2.2253856658935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,8191,2.213302421569824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,8191,2.2151103973388673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,8191,2.2127456665039062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,8191,0.41887521743774414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,8191,0.41653761863708494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,8191,2.2115583419799805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,128,1,4,16383,0.8173551559448242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,128,1,8,16383,0.8136992454528809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,8191,2.2119728088378907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,128,1,16,16383,0.8126864433288574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,128,1,32,16383,0.8119119644165039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,128,1,64,16383,0.8129952430725098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,128,1,2,16383,0.8231679916381835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,128,1,1,16383,0.9420495986938476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,1,0.05481280088424682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,1,0.02803199887275696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,1,0.024396799504756927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,1,0.02325280010700226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,1,0.022771200537681578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,1,0.0226623997092247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,128,1,2,16383,4.388550567626953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,128,1,1,16383,4.513873672485351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,1,0.06269919872283936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,1,0.03949599862098694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,128,1,4,16383,4.398540878295899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,128,1,8,16383,4.388281631469726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,128,1,16,16383,4.377350234985352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,1,0.028948798775672913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,128,1,32,16383,4.370286560058593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,1,0.028310400247573853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,3,0.052193599939346316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,1,0.028356799483299257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,1,0.028401601314544677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,3,0.022753599286079406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,3,0.02322240024805069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,3,0.022086399793624877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,3,0.027663999795913698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,3,0.022227199375629426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,3,0.06017919778823853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,3,0.029023998975753786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,3,0.02980160117149353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,3,0.028857600688934327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,3,0.028649601340293884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,7,0.052292799949645995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,7,0.027374398708343507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,7,0.02344159930944443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,7,0.02264160066843033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,7,0.02263360023498535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,7,0.022595199942588805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,7,0.06303840279579162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,7,0.039297598600387576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,7,0.03168480098247528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,7,0.030961599946022034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,128,1,64,16383,4.394950485229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,7,0.03091199994087219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,1,0.022729599475860597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,7,0.030817601084709167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,15,0.05297279953956604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,7,0.030668801069259642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,15,0.02850399911403656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,1,0.030003198981285097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,15,0.02298080027103424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,15,0.02263039946556091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,15,0.06702880263328552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,15,0.04448800086975098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,15,0.03473759889602661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,3,0.02274399995803833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,15,0.034083199501037595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,15,0.03383519947528839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,15,0.03385440111160278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,3,0.03598720133304596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,3,0.02906239926815033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,31,0.05446400046348572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,31,0.031225600838661195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,31,0.022867199778556824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,31,0.02285120040178299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,31,0.02284960001707077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,31,0.02256480008363724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,7,0.022540800273418427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,31,0.07292479872703553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,31,0.05108320116996765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,31,0.04224160015583038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,31,0.03984479904174805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,31,0.039419201016426084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,31,0.03904959857463837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,31,0.039073601365089417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,63,0.05778080224990845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,63,0.025975999236106873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,63,0.023423999547958374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,63,0.023817600309848787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,63,0.024110400676727296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,15,0.024198399484157564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,63,0.023948800563812257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,15,0.022716799378395082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,63,0.06918399930000305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,15,0.022623999416828154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,63,0.06215360164642334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,63,0.05508000254631042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,63,0.05440959930419922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,127,0.06613759994506836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,127,0.04209760129451752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,127,0.03714239895343781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,15,0.0337552011013031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,127,0.03245759904384613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,31,0.023464000225067137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,127,0.02971520125865936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,127,0.029238399863243104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,127,0.13000320196151732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,127,0.10163840055465698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,127,0.09837440252304078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,127,0.09620959758758545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,127,0.0954367995262146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,127,0.09501439929008484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,63,0.037396800518035886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,255,0.07967200279235839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,255,0.04765760004520416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,255,0.04671840071678161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,63,0.09132159948348999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,63,0.05667359828948974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,255,0.04739679992198944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,63,0.053908801078796385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,255,0.17785439491271973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,255,0.19961919784545898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,255,0.1723855972290039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,255,0.16961599588394166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,255,0.16943999528884887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,255,0.16815040111541749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,255,0.16817280054092407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,511,0.12780159711837769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,511,0.09421280026435852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,511,0.08464639782905578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,511,0.07826880216598511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,511,0.07698559761047363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,511,0.07652480006217957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,511,0.07413439750671387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,127,0.10668319463729858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,511,0.3599535942077637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,511,0.32642560005187987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,511,0.31818881034851076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,511,0.315012788772583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,511,0.3139695882797241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,511,0.31163361072540285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,511,0.31228320598602294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,1023,0.18492640256881715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,1023,0.13335360288619996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,255,0.05880799889564514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,1023,0.12456320524215699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,1023,0.1248736023902893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,255,0.05203199982643127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,1023,0.12478560209274292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,255,0.04573439955711365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,1023,0.6409552097320557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,1023,0.5939727783203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,1023,0.5871456146240235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,1023,0.5868720054626465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,127,0.03246400058269501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,1023,0.5838560104370117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,2047,0.29322879314422606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,2047,0.2414383888244629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,2047,0.23277759552001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,2047,0.22798559665679932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,2047,0.2245408058166504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,2047,0.2239936113357544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,1023,0.1433616042137146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,2047,1.1886480331420899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,2047,1.134438419342041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,1023,0.5988048076629638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,2047,1.131553554534912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,2047,1.1240127563476563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,2047,1.1270416259765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,4095,0.5344160079956055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,2047,1.1276911735534667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,2047,0.22415359020233155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,4095,0.44461917877197266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,4095,0.4299647808074951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,4095,0.45075201988220215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,4095,0.42090559005737305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,1023,0.12920479774475097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,4095,0.42002081871032715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,2047,1.1391839981079102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,1023,0.5859807968139649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,4095,2.297135925292969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,4095,2.242788887023926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,4095,2.2245840072631835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,4095,2.2177263259887696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,4095,0.42200961112976076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,256,1,1,8191,1.0015711784362793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,256,1,2,8191,0.8870847702026368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,4095,2.2241024017333983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,256,1,4,8191,0.8504624366760254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,256,1,8,8191,0.8186559677124023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,256,1,16,8191,0.8256591796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,256,1,32,8191,0.8245903968811035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,256,1,64,8191,0.8187567710876464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,4095,2.216075134277344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,4095,2.227804756164551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,1,0.09232479929924012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,1,0.05280159711837769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,1,0.041065600514411923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,1,0.03793599903583526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,1,0.03705439865589142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,1,0.036985599994659425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,1,0.03683040142059326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,256,1,2,8191,4.411524963378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,256,1,1,8191,4.485131072998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,1,0.05272160172462463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,1,0.044091200828552245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,256,1,4,8191,4.437961578369141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,256,1,16,8191,4.3869071960449215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,1,0.0441760003566742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,1,0.043161600828170776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,256,1,32,8191,4.395334243774414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,3,0.09673920273780823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,3,0.05485759973526001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,3,0.04208320081233978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,3,0.03702560067176819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,3,0.03686720132827759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,3,0.10264320373535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,3,0.06405119895935059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,256,1,64,8191,4.402783966064453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,3,0.05233759880065918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,3,0.04510239958763122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,3,0.04681119918823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,3,0.04514560103416443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,3,0.04524160027503967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,7,0.09294559955596923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,7,0.042305600643157956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,7,0.037852799892425536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,1,0.10055040121078491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,7,0.03684479892253876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,7,0.03699040114879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,7,0.036831998825073244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,7,0.10655679702758789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,1,0.062438398599624634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,7,0.05600320100784302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,7,0.0501151978969574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,7,0.0483711987733841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,7,0.04768320024013519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,256,1,8,8191,4.416625595092773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,7,0.04785119891166687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,1,0.043377599120140074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,15,0.09526079893112183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,15,0.04254559874534607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,15,0.03738240003585815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,15,0.056745600700378415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,15,0.03715679943561554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,15,0.03697920143604279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,15,0.07478399872779846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,15,0.1202623963356018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,15,0.056790399551391604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,15,0.05442879796028137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,15,0.053758400678634646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,15,0.05386080145835877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,3,0.03657439947128296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,31,0.09750080108642578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,31,0.05675680041313171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,31,0.04595040082931519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,31,0.038627201318740846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,31,0.0372624009847641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,31,0.03727680146694183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,31,0.037083199620246886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,31,0.13408639430999755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,31,0.08096160292625428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,31,0.07369279861450195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,31,0.07057920098304749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,31,0.06888480186462402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,31,0.0675216019153595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,63,0.10035519599914551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,7,0.05399199724197388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,63,0.05084480047225952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,63,0.0474480003118515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,63,0.04251199960708618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,63,0.04136320054531097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,7,0.06668639779090882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,63,0.1643936038017273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,63,0.12566879987716675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,63,0.11284639835357665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,63,0.11047519445419311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,63,0.10737600326538085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,15,0.03861280083656311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,63,0.1058575987815857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,63,0.10601279735565186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,15,0.06250720024108887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,127,0.11433919668197631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,127,0.07572640180587768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,127,0.06386240124702454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,127,0.058843201398849486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,3,0.03819839954376221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,127,0.05525919795036316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,127,0.054203200340271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,127,0.23387200832366944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,127,0.1849568009376526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,127,0.1807935953140259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,31,0.09152799844741821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,127,0.17650079727172852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,127,0.1759552001953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,255,0.1426416039466858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,63,0.0626031994819641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,255,0.0905888020992279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,255,0.08637760281562805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,63,0.043961599469184875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,255,0.0835536003112793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,255,0.08275359869003296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,255,0.3749727964401245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,255,0.33385438919067384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,255,0.3220047950744629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,255,0.31767680644989016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,255,0.3152528047561646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,255,0.3136192083358765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,127,0.05825120210647583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,255,0.31305599212646484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,511,0.23857760429382324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,511,0.17352160215377807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,127,0.1944543957710266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,511,0.15512640476226808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,127,0.17709920406341553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,511,0.14453760385513306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,511,0.1376415967941284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,255,0.10343359708786011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,511,0.6921711921691894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,511,0.6118639945983887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,511,0.6032320022583008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,255,0.08263199925422668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,511,0.6002399921417236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,511,0.5965871810913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,1023,0.35110719203948976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,1023,0.2530224084854126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,1023,0.2427056074142456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,1023,0.2371903896331787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,1023,0.23668160438537597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,1023,0.23199839591979982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,511,0.14078880548477174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,1023,1.1734736442565918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,1023,1.2468735694885253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,1023,1.1505536079406737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,511,0.6326367855072021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,1023,1.1520000457763673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,1023,0.2695696115493774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,1023,1.1429023742675781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,1023,1.136087989807129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,1023,1.132430362701416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,2047,0.5842703819274903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,2047,0.4667407989501953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,2047,0.4483391761779785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,2047,0.4399375915527344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,2047,0.43036642074584963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,2047,0.45844640731811526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,511,0.13876320123672486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,511,0.5964352130889893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,2047,2.371067237854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,2047,2.236639976501465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,2047,2.260095977783203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,2047,2.234820747375488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,2047,2.2257776260375977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,512,1,8,4095,0.8332768440246582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,512,1,4,4095,0.8434415817260742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,512,1,1,4095,1.077195167541504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,4095,0.8259504318237305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,2047,2.2336591720581054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,512,1,32,4095,0.8241007804870606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,2047,2.224216079711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,512,1,64,4095,0.8233167648315429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,512,1,16,2047,0.43355841636657716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,512,1,2,4095,0.8648159980773926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,1,0.17068320512771606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,1,0.09192799925804138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,1,0.07369279861450195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,1,0.06564800143241882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,1,0.06137279868125915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,1,0.06069120168685913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,1,0.06004160046577454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,1,0.17235200405120848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,512,1,2,4095,4.420113754272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,1,0.10008159875869752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,512,1,1,4095,4.650030517578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,1,0.07709439992904663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,512,1,8,4095,4.3856689453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,512,1,4,4095,4.422825622558594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,1,0.06899679899215698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,1,0.07261760234832763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,512,1,16,4095,4.402540969848633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,1,0.06916319727897643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,3,0.16417920589447021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,3,0.0607200026512146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,3,0.06612480282783509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,3,0.06034719944000244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,3,0.06025760173797608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,512,1,64,4095,4.419424057006836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,3,0.10708479881286621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,3,0.08118079900741577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,3,0.09106720089912415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,3,0.0734224021434784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,3,0.07135199904441833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,7,0.16656479835510254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,7,0.0900767982006073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,7,0.07458239793777466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,7,0.06603519916534424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,7,0.06112160086631775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,7,0.060433602333068846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,7,0.05975679755210876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,7,0.18615039587020873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,7,0.11417280435562134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,1,0.08635200262069702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,512,1,32,4095,4.410551834106445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,7,0.09593440294265747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,7,0.08765439987182617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,3,0.09532639980316163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,15,0.1719472050666809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,3,0.17768160104751587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,15,0.09378880262374878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,15,0.0754368007183075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,15,0.06741600036621094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,15,0.06160479784011841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,15,0.062196797132492064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,15,0.06091679930686951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,3,0.07202079892158508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,15,0.20871360301971437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,15,0.11225119829177857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,15,0.10264320373535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,15,0.09869279861450195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,15,0.09606080055236817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,31,0.17422560453414918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,31,0.09713119864463807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,31,0.078302401304245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,31,0.06979680061340332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,31,0.06484479904174804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,31,0.06259679794311523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,31,0.06104959845542908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,7,0.07822399735450744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,7,0.07746080160140992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,31,0.16298240423202515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,31,0.14315680265426636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,31,0.1337407946586609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,31,0.1289247989654541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,31,0.12702720165252684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,31,0.12616319656372071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,63,0.17785760164260864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,63,0.07730559706687927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,63,0.08560960292816162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,15,0.13070080280303956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,63,0.11442719697952271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,63,0.07188640236854553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,63,0.0712336003780365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,15,0.09265440106391906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,63,0.3000447988510132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,63,0.206166410446167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,63,0.1928272008895874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,63,0.19234559535980225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,63,0.19215840101242065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,127,0.20407679080963134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,127,0.11097120046615601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,127,0.09999679923057556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,127,0.09529920220375061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,7,0.0831712007522583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,127,0.09213759899139404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,127,0.09129120111465454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,127,0.4364768028259277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,127,0.367907190322876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,127,0.3441263914108276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,127,0.33371520042419434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,63,0.07329599857330323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,127,0.3270751953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,63,0.2291167974472046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,127,0.321616005897522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,255,0.26138720512390134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,63,0.19857439994812012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,255,0.18392000198364258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,255,0.1496880054473877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,255,0.1408959984779358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,255,0.14293440580368041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,127,0.13381919860839844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,255,0.7148367881774902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,255,0.6431312084197998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,3,0.07653120160102844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,31,0.2405616044998169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,255,0.6140367984771729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,255,0.602729606628418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,255,0.6013584136962891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,255,0.5951056003570556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,511,0.44098720550537107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,511,0.3198911905288696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,511,0.28118879795074464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,127,0.3255743980407715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,511,0.2625135898590088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,511,0.2574912071228027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,255,0.15923360586166382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,511,0.25241119861602784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,511,0.2521552085876465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,255,0.14337919950485228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,511,1.3400992393493651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,511,1.1624095916748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,511,1.1841296195983886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,511,1.1596960067749023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,255,0.5950175762176514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,1023,0.5164239883422852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,1023,0.48115200996398927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,1023,0.6732079982757568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,511,1.1579744338989257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,1023,0.46280641555786134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,1023,0.45140957832336426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,1023,0.44693760871887206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,1023,0.46383042335510255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,511,1.2261391639709474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,511,1.1622495651245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,1023,2.4589279174804686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,1023,2.30914249420166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,1023,2.268035125732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,1023,2.2663055419921876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,128,1024,1,1,2047,1.159388828277588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,1023,2.2409759521484376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,1023,2.2432559967041015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,8,1024,1,16,2047,0.845246410369873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,32,1024,1,4,2047,0.8800975799560546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,64,1024,1,2,2047,0.9511936187744141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,4,1024,1,32,2047,0.9163632392883301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,2,1024,1,64,2047,0.8382224082946778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,1023,2.2529008865356444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,bfloat16,16,1024,1,8,2047,0.857975959777832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,64,1024,1,2,2047,4.482888031005859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,16,1024,1,8,2047,4.443636703491211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,128,1024,1,1,2047,4.68169937133789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,8,1024,1,16,2047,4.460737609863282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,4,1024,1,32,2047,4.400547027587891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,2,1024,1,64,2047,4.430940628051758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_generation,flash_attention,bfloat16,fp8,32,1024,1,4,2047,4.4770862579345705
