==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=humaneval  mode=greedy
commit=52993e4  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 23.955334966652856
  tokens_per_second_p25 = 22.118394981825364
  tokens_per_second_p75 = 24.43403990022169
  tokens_per_second_mean = 23.377266893261663
  acceptance_rate_mean = 0.7748324561339393
  gpu_memory_gb = 3.9242053031921387
  speedup_vs_flashspec_compiled_mean = 0.33010039819332226
  speedup_vs_flashspec_compiled_median = 0.32885905007214555
  speedup_vs_flashspec_compiled_std = 0.0054629605500696185
  speedup_vs_hf_assisted_mean = 1.0757625853656174
  speedup_vs_hf_assisted_median = 1.0609567223465555
  speedup_vs_hf_assisted_std = 0.08281312119646546
  speedup_vs_hf_greedy_compiled_mean = 0.47554628790423054
  speedup_vs_hf_greedy_compiled_median = 0.48782041481421456
  speedup_vs_hf_greedy_compiled_std = 0.036972664527631025
  speedup_vs_hf_greedy_mean = 0.9934924598148624
  speedup_vs_hf_greedy_median = 1.0100837739015707
  speedup_vs_hf_greedy_std = 0.07427246922725399
  forward_counts.drafter_forwards = 1230
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 4.19672131147541
  forward_counts.n_generations = 10
  forward_counts.n_steps = 305
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 315
  forward_counts.target_forwards_per_emitted_token = 0.24609375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0019037710845964725
  step_profile.bookkeeping_ms_mean = 0.3438650750234479
  step_profile.bookkeeping_ms_total = 104.8788478821516
  step_profile.draft_fraction = 0.7719555360610002
  step_profile.draft_ms_mean = 139.43301821849386
  step_profile.draft_ms_total = 42527.070556640625
  step_profile.n_steps = 305.0
  step_profile.sample_fraction = 0.0008218625018387269
  step_profile.sample_ms_mean = 0.14844737013832468
  step_profile.sample_ms_total = 45.276447892189026
  step_profile.step_ms_mean = 180.62312102840573
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 4.19672131147541
  step_profile.verify_fraction = 0.22531883035256461
  step_profile.verify_ms_mean = 40.6977903647501
  step_profile.verify_ms_total = 12412.82606124878
  host_syncs.by_kind = {'cuda': 38}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 36
  host_syncs.per_step = 1.0555555555555556
  host_syncs.per_target_forward = 1.0555555555555556
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 36
  host_syncs.total = 38
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 145
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 37
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[flashspec_compiled]
  tokens_per_second_median = 72.82514637900351
  tokens_per_second_p25 = 66.56029254824399
  tokens_per_second_p75 = 74.62784772525018
  tokens_per_second_mean = 70.80594967447391
  acceptance_rate_mean = 0.7674472295086122
  gpu_memory_gb = 3.9391956329345703
  speedup_vs_flashspec_ucb_mean = 3.030202740912143
  speedup_vs_flashspec_ucb_median = 3.040844381198677
  speedup_vs_flashspec_ucb_std = 0.04963895478722407
  speedup_vs_hf_assisted_mean = 3.258189766208409
  speedup_vs_hf_assisted_median = 3.266906470700773
  speedup_vs_hf_assisted_std = 0.23513322192593836
  speedup_vs_hf_greedy_compiled_mean = 1.4403391620430996
  speedup_vs_hf_greedy_compiled_median = 1.4858433502745476
  speedup_vs_hf_greedy_compiled_std = 0.1062220714437251
  speedup_vs_hf_greedy_mean = 3.0093972988044415
  speedup_vs_hf_greedy_median = 3.07061233431369
  speedup_vs_hf_greedy_std = 0.21584386850536136
  forward_counts.drafter_forwards = 1238
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 4.169381107491857
  forward_counts.n_generations = 10
  forward_counts.n_steps = 307
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 317
  forward_counts.target_forwards_per_emitted_token = 0.24765625
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.005914694039726373
  step_profile.bookkeeping_ms_mean = 0.34993407135095195
  step_profile.bookkeeping_ms_total = 107.42975990474224
  step_profile.draft_fraction = 0.6577187913268348
  step_profile.draft_ms_mean = 38.91295355383659
  step_profile.draft_ms_total = 11946.276741027832
  step_profile.n_steps = 307.0
  step_profile.sample_fraction = 0.0026253843409469566
  step_profile.sample_ms_mean = 0.15532695776282382
  step_profile.sample_ms_total = 47.68537603318691
  step_profile.step_ms_mean = 59.16351192480966
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 4.169381107491857
  step_profile.verify_fraction = 0.3337411302924919
  step_profile.verify_ms_mean = 19.745297341859303
  step_profile.verify_ms_total = 6061.806283950806
  host_syncs.by_kind = {'cuda': 40}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 36
  host_syncs.per_step = 1.1111111111111112
  host_syncs.per_target_forward = 1.1111111111111112
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 36
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 40
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, 94, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 9
  greedy_token_match.reference = hf_greedy
[hf_assisted]
  tokens_per_second_median = 22.237121132281878
  tokens_per_second_p25 = 20.731428317512226
  tokens_per_second_p75 = 23.083469585054274
  tokens_per_second_mean = 21.846467981152262
  gpu_memory_gb = 4.012396335601807
  forward_counts.drafter_forwards = 1385
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 7.231638418079096
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 187
  forward_counts.target_forwards_per_emitted_token = 0.14609375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 617}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 24.68
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 25
  host_syncs.total = 617
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 153
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 26
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[hf_greedy]
  tokens_per_second_median = 23.47412238102048
  tokens_per_second_p25 = 23.227123999475463
  tokens_per_second_p75 = 23.77857377434369
  tokens_per_second_mean = 23.526828207785346
  gpu_memory_gb = 3.9314565658569336
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_greedy_compiled]
  tokens_per_second_median = 49.12146246181767
  tokens_per_second_p25 = 49.063005875735676
  tokens_per_second_p75 = 49.28637488375219
  tokens_per_second_mean = 49.158064750286485
  gpu_memory_gb = 3.9255499839782715
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, 94, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 9
  greedy_token_match.reference = hf_greedy

==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=humaneval  mode=sample T=0.7
commit=52993e4  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 20.926996555037533
  tokens_per_second_p25 = 19.006980713893356
  tokens_per_second_p75 = 22.107230498003904
  tokens_per_second_mean = 20.381163003890865
  acceptance_rate_mean = 0.6882722339966006
  gpu_memory_gb = 3.945862293243408
  speedup_vs_flashspec_compiled_mean = 0.3202859875204197
  speedup_vs_flashspec_compiled_median = 0.3201268770489362
  speedup_vs_flashspec_compiled_std = 0.025162655322136098
  speedup_vs_hf_assisted_mean = 1.0809679600750821
  speedup_vs_hf_assisted_median = 1.0763259822470552
  speedup_vs_hf_assisted_std = 0.12701641205311331
  speedup_vs_hf_sample_compiled_mean = 0.4158241958798537
  speedup_vs_hf_sample_compiled_median = 0.4261660719179908
  speedup_vs_hf_sample_compiled_std = 0.05395100822894165
  speedup_vs_hf_sample_mean = 0.8622114410954593
  speedup_vs_hf_sample_median = 0.8889351997557875
  speedup_vs_hf_sample_std = 0.11406144068620855
  forward_counts.drafter_forwards = 1390
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.710144927536232
  forward_counts.n_generations = 10
  forward_counts.n_steps = 345
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 355
  forward_counts.target_forwards_per_emitted_token = 0.27734375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0022480055162598877
  step_profile.bookkeeping_ms_mean = 0.41664834765420444
  step_profile.bookkeeping_ms_total = 143.74367994070053
  step_profile.draft_fraction = 0.7697398838834666
  step_profile.draft_ms_mean = 142.66461911408797
  step_profile.draft_ms_total = 49219.29359436035
  step_profile.n_steps = 345.0
  step_profile.sample_fraction = 0.002415020891927614
  step_profile.sample_ms_mean = 0.4476032006567803
  step_profile.sample_ms_total = 154.4231042265892
  step_profile.step_ms_mean = 185.34133686086406
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.710144927536232
  step_profile.verify_fraction = 0.22559708970834585
  step_profile.verify_ms_mean = 41.812466198465096
  step_profile.verify_ms_total = 14425.300838470459
  host_syncs.by_kind = {'cuda': 37}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 35
  host_syncs.per_step = 1.0571428571428572
  host_syncs.per_target_forward = 1.0571428571428572
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 35
  host_syncs.total = 37
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 141
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 36
[flashspec_compiled]
  tokens_per_second_median = 64.65023157407026
  tokens_per_second_p25 = 58.88331145207276
  tokens_per_second_p75 = 67.30609587744007
  tokens_per_second_mean = 63.483100389122626
  acceptance_rate_mean = 0.7018560004266362
  gpu_memory_gb = 3.945862293243408
  speedup_vs_flashspec_ucb_mean = 3.1424386786788667
  speedup_vs_flashspec_ucb_median = 3.1237669703114515
  speedup_vs_flashspec_ucb_std = 0.2603924266601755
  speedup_vs_hf_assisted_mean = 3.3855737766588745
  speedup_vs_hf_assisted_median = 3.3484703254583867
  speedup_vs_hf_assisted_std = 0.39504869853856245
  speedup_vs_hf_sample_compiled_mean = 1.2950403037854747
  speedup_vs_hf_sample_compiled_median = 1.320689325997725
  speedup_vs_hf_sample_compiled_std = 0.10515182476332334
  speedup_vs_hf_sample_mean = 2.6856050959875404
  speedup_vs_hf_sample_median = 2.7423863913637065
  speedup_vs_hf_sample_std = 0.2337952364219114
  forward_counts.drafter_forwards = 1370
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.764705882352941
  forward_counts.n_generations = 10
  forward_counts.n_steps = 340
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 350
  forward_counts.target_forwards_per_emitted_token = 0.2734375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.006907362689384729
  step_profile.bookkeeping_ms_mean = 0.4119171774562667
  step_profile.bookkeeping_ms_total = 140.0518403351307
  step_profile.draft_fraction = 0.6578513463227282
  step_profile.draft_ms_mean = 39.230641555786136
  step_profile.draft_ms_total = 13338.418128967285
  step_profile.n_steps = 340.0
  step_profile.sample_fraction = 0.0066157920430872045
  step_profile.sample_ms_mean = 0.3945295053369859
  step_profile.sample_ms_total = 134.1400318145752
  step_profile.step_ms_mean = 59.63450827467091
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.764705882352941
  step_profile.verify_fraction = 0.32862549894479987
  step_profile.verify_ms_mean = 19.597420036091524
  step_profile.verify_ms_total = 6663.122812271118
  host_syncs.by_kind = {'cuda': 39}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 35
  host_syncs.per_step = 1.1142857142857143
  host_syncs.per_target_forward = 1.1142857142857143
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 35
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 39
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
[hf_assisted]
  tokens_per_second_median = 18.252091267763525
  tokens_per_second_p25 = 17.25533897621466
  tokens_per_second_p75 = 20.413537204874352
  tokens_per_second_mean = 18.957413821849507
  gpu_memory_gb = 4.025937080383301
  forward_counts.drafter_forwards = 1565
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 6.037735849056604
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 222
  forward_counts.target_forwards_per_emitted_token = 0.1734375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 911}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 29.387096774193548
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 31
  host_syncs.total = 911
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 185
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 32
[hf_sample]
  tokens_per_second_median = 23.737994536255016
  tokens_per_second_p25 = 23.581683402366696
  tokens_per_second_p75 = 23.790908382598783
  tokens_per_second_mean = 23.65519734504495
  gpu_memory_gb = 3.9465904235839844
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_sample_compiled]
  tokens_per_second_median = 49.040621806485035
  tokens_per_second_p25 = 48.91420681457233
  tokens_per_second_p75 = 49.127526309195396
  tokens_per_second_mean = 49.026357617491875
  gpu_memory_gb = 3.939551830291748
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile

==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=mt_bench  mode=greedy
commit=52993e4  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 17.848947424797554
  tokens_per_second_p25 = 17.05064438910831
  tokens_per_second_p75 = 19.216907014764494
  tokens_per_second_mean = 18.212446169934015
  acceptance_rate_mean = 0.5868378012374362
  gpu_memory_gb = 3.905423641204834
  speedup_vs_flashspec_compiled_mean = 0.3263977099331733
  speedup_vs_flashspec_compiled_median = 0.3177471432768646
  speedup_vs_flashspec_compiled_std = 0.02703929798989694
  speedup_vs_hf_assisted_mean = 1.1275125593375939
  speedup_vs_hf_assisted_median = 1.082497949211112
  speedup_vs_hf_assisted_std = 0.22765525888205237
  speedup_vs_hf_greedy_compiled_mean = 0.37099958080467416
  speedup_vs_hf_greedy_compiled_median = 0.36444516173510544
  speedup_vs_hf_greedy_compiled_std = 0.05567486947051054
  speedup_vs_hf_greedy_mean = 0.7812990140987142
  speedup_vs_hf_greedy_median = 0.7712457435366594
  speedup_vs_hf_greedy_std = 0.11231621347309953
  forward_counts.drafter_forwards = 1578
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.2653061224489797
  forward_counts.n_generations = 10
  forward_counts.n_steps = 392
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 402
  forward_counts.target_forwards_per_emitted_token = 0.3140625
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0026882709174687776
  step_profile.bookkeeping_ms_mean = 0.4923061221761971
  step_profile.bookkeeping_ms_total = 192.98399989306927
  step_profile.draft_fraction = 0.7698535678271795
  step_profile.draft_ms_mean = 140.9841627782705
  step_profile.draft_ms_total = 55265.79180908203
  step_profile.n_steps = 392.0
  step_profile.sample_fraction = 0.00083997751260875
  step_profile.sample_ms_mean = 0.15382604084300752
  step_profile.sample_ms_total = 60.299808010458946
  step_profile.step_ms_mean = 183.13114164838072
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.2653061224489797
  step_profile.verify_fraction = 0.226618183742743
  step_profile.verify_ms_mean = 41.50084670709104
  step_profile.verify_ms_total = 16268.331909179688
  host_syncs.by_kind = {'cuda': 47}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 44
  host_syncs.per_step = 1.0681818181818181
  host_syncs.per_target_forward = 1.0681818181818181
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 44
  host_syncs.total = 47
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 177
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 45
  greedy_token_match.first_divergence = [None, 44, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 9
  greedy_token_match.reference = hf_greedy
[flashspec_compiled]
  tokens_per_second_median = 55.252001625954684
  tokens_per_second_p25 = 51.03451752732001
  tokens_per_second_p75 = 56.23720752136681
  tokens_per_second_mean = 55.963676154917366
  acceptance_rate_mean = 0.5310738087770345
  gpu_memory_gb = 3.9152259826660156
  speedup_vs_flashspec_ucb_mean = 3.0811351748402753
  speedup_vs_flashspec_ucb_median = 3.1471566033210427
  speedup_vs_flashspec_ucb_std = 0.2099798433692188
  speedup_vs_hf_assisted_mean = 3.4779518896990553
  speedup_vs_hf_assisted_median = 3.2676885098898643
  speedup_vs_hf_assisted_std = 0.7733865457137848
  speedup_vs_hf_greedy_compiled_mean = 1.1399292915524684
  speedup_vs_hf_greedy_compiled_median = 1.1245701242847757
  speedup_vs_hf_greedy_compiled_std = 0.17279032013984028
  speedup_vs_hf_greedy_mean = 2.401983217699875
  speedup_vs_hf_greedy_median = 2.379455989273368
  speedup_vs_hf_greedy_std = 0.357017672181821
  forward_counts.drafter_forwards = 1618
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.18407960199005
  forward_counts.n_generations = 10
  forward_counts.n_steps = 402
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 412
  forward_counts.target_forwards_per_emitted_token = 0.321875
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.008739650639202082
  step_profile.bookkeeping_ms_mean = 0.5073160583255303
  step_profile.bookkeeping_ms_total = 203.94105544686317
  step_profile.draft_fraction = 0.6540452601209293
  step_profile.draft_ms_mean = 37.96578113119401
  step_profile.draft_ms_total = 15262.24401473999
  step_profile.n_steps = 402.0
  step_profile.sample_fraction = 0.002761754382352098
  step_profile.sample_ms_mean = 0.1603133128724288
  step_profile.sample_ms_total = 64.44595177471638
  step_profile.step_ms_mean = 58.047635914637375
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.18407960199005
  step_profile.verify_fraction = 0.33445333485751644
  step_profile.verify_ms_mean = 19.414225412245415
  step_profile.verify_ms_total = 7804.518615722656
  host_syncs.by_kind = {'cuda': 48}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 44
  host_syncs.per_step = 1.0909090909090908
  host_syncs.per_target_forward = 1.0909090909090908
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 44
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 48
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[hf_assisted]
  tokens_per_second_median = 18.523353839382736
  tokens_per_second_p25 = 13.817359096283546
  tokens_per_second_p75 = 19.874466083142597
  tokens_per_second_mean = 16.898456041291062
  gpu_memory_gb = 3.9850401878356934
  forward_counts.drafter_forwards = 1803
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.8905775075987843
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 339
  forward_counts.target_forwards_per_emitted_token = 0.26484375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 1184}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 28.878048780487806
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 41
  host_syncs.total = 1184
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 310
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 42
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[hf_greedy]
  tokens_per_second_median = 23.397178479769785
  tokens_per_second_p25 = 22.921001536069262
  tokens_per_second_p75 = 23.552116432251463
  tokens_per_second_mean = 23.29047897726066
  gpu_memory_gb = 3.9156312942504883
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_greedy_compiled]
  tokens_per_second_median = 49.12532116486909
  tokens_per_second_p25 = 48.97701107926284
  tokens_per_second_p75 = 49.21500203797986
  tokens_per_second_mean = 49.084991214708566
  gpu_memory_gb = 3.910264492034912
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy

==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=mt_bench  mode=sample T=0.7
commit=52993e4  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 18.051633117833564
  tokens_per_second_p25 = 16.19958313691243
  tokens_per_second_p75 = 18.82464363903081
  tokens_per_second_mean = 17.99347290909512
  acceptance_rate_mean = 0.5437414602239845
  gpu_memory_gb = 3.923186779022217
  speedup_vs_flashspec_compiled_mean = 0.31982592415552624
  speedup_vs_flashspec_compiled_median = 0.31517072053162654
  speedup_vs_flashspec_compiled_std = 0.016197728012732945
  speedup_vs_hf_assisted_mean = 1.5701057498902078
  speedup_vs_hf_assisted_median = 1.246618707574589
  speedup_vs_hf_assisted_std = 0.6197416201154381
  speedup_vs_hf_sample_compiled_mean = 0.3676141154238134
  speedup_vs_hf_sample_compiled_median = 0.3684109285057109
  speedup_vs_hf_sample_compiled_std = 0.048925934817909376
  speedup_vs_hf_sample_mean = 0.7827767956232581
  speedup_vs_hf_sample_median = 0.7831004034693231
  speedup_vs_hf_sample_std = 0.10510731186707156
  forward_counts.drafter_forwards = 1546
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.3333333333333335
  forward_counts.n_generations = 10
  forward_counts.n_steps = 384
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 394
  forward_counts.target_forwards_per_emitted_token = 0.3078125
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.002644784808608365
  step_profile.bookkeeping_ms_mean = 0.4978414997458458
  step_profile.bookkeeping_ms_total = 191.17113590240479
  step_profile.draft_fraction = 0.7703810467523626
  step_profile.draft_ms_mean = 145.01280196507773
  step_profile.draft_ms_total = 55684.915954589844
  step_profile.n_steps = 384.0
  step_profile.sample_fraction = 0.0024928175412315266
  step_profile.sample_ms_mean = 0.4692359163891524
  step_profile.sample_ms_total = 180.18659189343452
  step_profile.step_ms_mean = 188.23516307468526
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.3333333333333335
  step_profile.verify_fraction = 0.22448135089779744
  step_profile.verify_ms_mean = 42.25528369347254
  step_profile.verify_ms_total = 16226.028938293457
  host_syncs.by_kind = {'cuda': 48}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 46
  host_syncs.per_step = 1.0434782608695652
  host_syncs.per_target_forward = 1.0434782608695652
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 46
  host_syncs.total = 48
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 185
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 47
[flashspec_compiled]
  tokens_per_second_median = 55.877849299137125
  tokens_per_second_p25 = 50.58062889171737
  tokens_per_second_p75 = 59.43849884307497
  tokens_per_second_mean = 56.25675041132389
  acceptance_rate_mean = 0.5374968568894605
  gpu_memory_gb = 3.9273171424865723
  speedup_vs_flashspec_ucb_mean = 3.134059136863989
  speedup_vs_flashspec_ucb_median = 3.1728836261336024
  speedup_vs_flashspec_ucb_std = 0.14547574745217234
  speedup_vs_hf_assisted_mean = 4.883803818839702
  speedup_vs_hf_assisted_median = 4.0319731142311515
  speedup_vs_hf_assisted_std = 1.8420737943506686
  speedup_vs_hf_sample_compiled_mean = 1.1493423000654888
  speedup_vs_hf_sample_compiled_median = 1.1410430525952333
  speedup_vs_hf_sample_compiled_std = 0.13964981312596908
  speedup_vs_hf_sample_mean = 2.447146215174908
  speedup_vs_hf_sample_median = 2.4026007789116655
  speedup_vs_hf_sample_std = 0.2992352315348531
  forward_counts.drafter_forwards = 1562
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.2989690721649483
  forward_counts.n_generations = 10
  forward_counts.n_steps = 388
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 398
  forward_counts.target_forwards_per_emitted_token = 0.3109375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.008572496921251196
  step_profile.bookkeeping_ms_mean = 0.5092328250331363
  step_profile.bookkeeping_ms_total = 197.58233611285686
  step_profile.draft_fraction = 0.6566248200889545
  step_profile.draft_ms_mean = 39.00554473129744
  step_profile.draft_ms_total = 15134.151355743408
  step_profile.n_steps = 388.0
  step_profile.sample_fraction = 0.006754352815625679
  step_profile.sample_ms_mean = 0.4012294430861768
  step_profile.sample_ms_total = 155.6770239174366
  step_profile.step_ms_mean = 59.40309220418027
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.2989690721649483
  step_profile.verify_fraction = 0.3280483301741686
  step_profile.verify_ms_mean = 19.48708520476351
  step_profile.verify_ms_total = 7560.989059448242
  host_syncs.by_kind = {'cuda': 50}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 46
  host_syncs.per_step = 1.0869565217391304
  host_syncs.per_target_forward = 1.0869565217391304
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 46
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 50
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
[hf_assisted]
  tokens_per_second_median = 13.640262305103246
  tokens_per_second_p25 = 8.678840619024454
  tokens_per_second_p75 = 16.993490021453383
  tokens_per_second_mean = 13.303585312651354
  gpu_memory_gb = 4.008172988891602
  forward_counts.drafter_forwards = 2675
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.9751552795031055
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 332
  forward_counts.target_forwards_per_emitted_token = 0.259375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 1603}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 42.18421052631579
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 38
  host_syncs.total = 1603
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 388
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 39
[hf_sample]
  tokens_per_second_median = 22.970064276527687
  tokens_per_second_p25 = 22.871294494922637
  tokens_per_second_p75 = 23.08729168048587
  tokens_per_second_mean = 22.98840727987751
  gpu_memory_gb = 3.9295730590820312
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_sample_compiled]
  tokens_per_second_median = 48.94843360467012
  tokens_per_second_p25 = 48.86839827300831
  tokens_per_second_p75 = 49.071253152907715
  tokens_per_second_mean = 48.94016450014386
  gpu_memory_gb = 3.924206256866455
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile

