==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=humaneval  mode=greedy
commit=8d036d7  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 26.02536363267037
  tokens_per_second_p25 = 24.113137938821488
  tokens_per_second_p75 = 26.416962512055154
  tokens_per_second_mean = 25.2709286731536
  acceptance_rate_mean = 0.7748324561339393
  gpu_memory_gb = 3.9210314750671387
  speedup_vs_flashspec_compiled_mean = 0.3533339372170675
  speedup_vs_flashspec_compiled_median = 0.35288296308959055
  speedup_vs_flashspec_compiled_std = 0.005845669652469166
  speedup_vs_hf_assisted_mean = 1.0724329885068866
  speedup_vs_hf_assisted_median = 1.0318343857730912
  speedup_vs_hf_assisted_std = 0.07662863165251793
  speedup_vs_hf_greedy_compiled_mean = 0.5123979186743102
  speedup_vs_hf_greedy_compiled_median = 0.5282968591639221
  speedup_vs_hf_greedy_compiled_std = 0.03844833151385093
  speedup_vs_hf_greedy_mean = 0.9980519535364397
  speedup_vs_hf_greedy_median = 1.0193827466008485
  speedup_vs_hf_greedy_std = 0.07284306810582675
  forward_counts.drafter_forwards = 1230
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 4.19672131147541
  forward_counts.n_generations = 10
  forward_counts.n_steps = 305
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 315
  forward_counts.target_forwards_per_emitted_token = 0.24609375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0018587390212922114
  step_profile.bookkeeping_ms_mean = 0.3104378748624051
  step_profile.bookkeeping_ms_total = 94.68355183303356
  step_profile.draft_fraction = 0.7696617101608727
  step_profile.draft_ms_mean = 128.54528953677317
  step_profile.draft_ms_total = 39206.31330871582
  step_profile.n_steps = 305.0
  step_profile.sample_fraction = 0.0007802120485253866
  step_profile.sample_ms_mean = 0.1303073575750726
  step_profile.sample_ms_total = 39.74374406039715
  step_profile.step_ms_mean = 167.01531054455725
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 4.19672131147541
  step_profile.verify_fraction = 0.22769933876930967
  step_profile.verify_ms_mean = 38.0292757753466
  step_profile.verify_ms_total = 11598.929111480713
  host_syncs.by_kind = {'cuda': 38}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 36
  host_syncs.per_step = 1.0555555555555556
  host_syncs.per_target_forward = 1.0555555555555556
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 36
  host_syncs.total = 38
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 145
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 37
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[flashspec_compiled]
  tokens_per_second_median = 73.49117484423093
  tokens_per_second_p25 = 67.66749962983283
  tokens_per_second_p75 = 75.22223233061746
  tokens_per_second_mean = 71.53432087333763
  acceptance_rate_mean = 0.7674472295086122
  gpu_memory_gb = 3.9358386993408203
  speedup_vs_flashspec_ucb_mean = 2.830944471348561
  speedup_vs_flashspec_ucb_median = 2.8338175395282628
  speedup_vs_flashspec_ucb_std = 0.04598643009013109
  speedup_vs_hf_assisted_mean = 3.0362177400376407
  speedup_vs_hf_assisted_median = 2.968039793678412
  speedup_vs_hf_assisted_std = 0.22444777462982243
  speedup_vs_hf_greedy_compiled_mean = 1.450394922632253
  speedup_vs_hf_greedy_compiled_median = 1.4929247183661758
  speedup_vs_hf_greedy_compiled_std = 0.10986743111464027
  speedup_vs_hf_greedy_mean = 2.8249681818768515
  speedup_vs_hf_greedy_median = 2.868400153448608
  speedup_vs_hf_greedy_std = 0.20691456029554223
  forward_counts.drafter_forwards = 1238
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 4.169381107491857
  forward_counts.n_generations = 10
  forward_counts.n_steps = 307
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 317
  forward_counts.target_forwards_per_emitted_token = 0.24765625
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.005504946952400353
  step_profile.bookkeeping_ms_mean = 0.32249120537931836
  step_profile.bookkeeping_ms_total = 99.00480005145073
  step_profile.draft_fraction = 0.6585451205278279
  step_profile.draft_ms_mean = 38.57893846244688
  step_profile.draft_ms_total = 11843.734107971191
  step_profile.n_steps = 307.0
  step_profile.sample_fraction = 0.0024814412598239636
  step_profile.sample_ms_mean = 0.145367973547805
  step_profile.sample_ms_total = 44.62796787917614
  step_profile.step_ms_mean = 58.58207320938864
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 4.169381107491857
  step_profile.verify_fraction = 0.33346849125994776
  step_profile.verify_ms_mean = 19.535275568014637
  step_profile.verify_ms_total = 5997.329599380493
  host_syncs.by_kind = {'cuda': 40}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 36
  host_syncs.per_step = 1.1111111111111112
  host_syncs.per_target_forward = 1.1111111111111112
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 36
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 40
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, 94, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 9
  greedy_token_match.reference = hf_greedy
  greedy_token_match_compiled.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match_compiled.n_prompts = 10
  greedy_token_match_compiled.prompts_matched = 10
  greedy_token_match_compiled.reference = hf_greedy_compiled
[hf_assisted]
  tokens_per_second_median = 24.19664417567833
  tokens_per_second_p25 = 22.245649238351376
  tokens_per_second_p75 = 25.200750728941774
  tokens_per_second_mean = 23.670070178990724
  gpu_memory_gb = 4.010560035705566
  forward_counts.drafter_forwards = 1385
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 7.231638418079096
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 187
  forward_counts.target_forwards_per_emitted_token = 0.14609375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 617}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 24.68
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 25
  host_syncs.total = 617
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 153
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 26
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[hf_greedy]
  tokens_per_second_median = 25.272922944874917
  tokens_per_second_p25 = 25.064466964565117
  tokens_per_second_p75 = 25.500231713400083
  tokens_per_second_mean = 25.320169143224767
  gpu_memory_gb = 3.9280996322631836
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_greedy_compiled]
  tokens_per_second_median = 49.27609336228632
  tokens_per_second_p25 = 49.1847500987219
  tokens_per_second_p75 = 49.34777025069719
  tokens_per_second_mean = 49.321943829106125
  gpu_memory_gb = 3.922210216522217
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, 94, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 9
  greedy_token_match.reference = hf_greedy

==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=humaneval  mode=sample T=0.7
commit=8d036d7  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 23.054835447878503
  tokens_per_second_p25 = 21.110460201553547
  tokens_per_second_p75 = 24.779995491896514
  tokens_per_second_mean = 22.65499383558366
  acceptance_rate_mean = 0.6882722339966006
  gpu_memory_gb = 3.9422059059143066
  speedup_vs_flashspec_compiled_mean = 0.35190281342909524
  speedup_vs_flashspec_compiled_median = 0.3525663970748343
  speedup_vs_flashspec_compiled_std = 0.02537062411977162
  speedup_vs_hf_assisted_mean = 1.0879824513890957
  speedup_vs_hf_assisted_median = 1.0736463665588767
  speedup_vs_hf_assisted_std = 0.13229481251177053
  speedup_vs_hf_sample_compiled_mean = 0.4598882818842379
  speedup_vs_hf_sample_compiled_median = 0.46804709580075965
  speedup_vs_hf_sample_compiled_std = 0.05745973698626698
  speedup_vs_hf_sample_mean = 0.8900382007430219
  speedup_vs_hf_sample_median = 0.9109461711070447
  speedup_vs_hf_sample_std = 0.11303332217376089
  forward_counts.drafter_forwards = 1390
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.710144927536232
  forward_counts.n_generations = 10
  forward_counts.n_steps = 345
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 355
  forward_counts.target_forwards_per_emitted_token = 0.27734375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.002210732333907
  step_profile.bookkeeping_ms_mean = 0.36840598280879033
  step_profile.bookkeeping_ms_total = 127.10006406903267
  step_profile.draft_fraction = 0.7719396960610024
  step_profile.draft_ms_mean = 128.63936444710995
  step_profile.draft_ms_total = 44380.58073425293
  step_profile.n_steps = 345.0
  step_profile.sample_fraction = 0.002224247050842795
  step_profile.sample_ms_mean = 0.3706581336000691
  step_profile.sample_ms_total = 127.87705609202385
  step_profile.step_ms_mean = 166.64431833668033
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.710144927536232
  step_profile.verify_fraction = 0.22362532455424775
  step_profile.verify_ms_mean = 37.26588977316152
  step_profile.verify_ms_total = 12856.731971740723
  host_syncs.by_kind = {'cuda': 37}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 35
  host_syncs.per_step = 1.0571428571428572
  host_syncs.per_target_forward = 1.0571428571428572
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 35
  host_syncs.total = 37
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 141
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 36
[flashspec_compiled]
  tokens_per_second_median = 65.40179447333742
  tokens_per_second_p25 = 59.35656427020519
  tokens_per_second_p75 = 68.5928206976523
  tokens_per_second_mean = 64.22953072922768
  acceptance_rate_mean = 0.7018560004266362
  gpu_memory_gb = 3.9422059059143066
  speedup_vs_flashspec_ucb_mean = 2.8574593444097443
  speedup_vs_flashspec_ucb_median = 2.8363542447229317
  speedup_vs_flashspec_ucb_std = 0.22071648377148556
  speedup_vs_hf_assisted_mean = 3.0995616233893997
  speedup_vs_hf_assisted_median = 3.06365302315802
  speedup_vs_hf_assisted_std = 0.37040543619427285
  speedup_vs_hf_sample_compiled_mean = 1.3038989435097832
  speedup_vs_hf_sample_compiled_median = 1.326870202978973
  speedup_vs_hf_sample_compiled_std = 0.10728795030275629
  speedup_vs_hf_sample_mean = 2.5236358201951843
  speedup_vs_hf_sample_median = 2.562178271321323
  speedup_vs_hf_sample_std = 0.21881054033180508
  forward_counts.drafter_forwards = 1370
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.764705882352941
  forward_counts.n_generations = 10
  forward_counts.n_steps = 340
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 350
  forward_counts.target_forwards_per_emitted_token = 0.2734375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0064683766353118705
  step_profile.bookkeeping_ms_mean = 0.3814550593495369
  step_profile.bookkeeping_ms_total = 129.69472017884254
  step_profile.draft_fraction = 0.6571434948155445
  step_profile.draft_ms_mean = 38.75326452816234
  step_profile.draft_ms_total = 13176.109939575195
  step_profile.n_steps = 340.0
  step_profile.sample_fraction = 0.006363673432078251
  step_profile.sample_ms_mean = 0.3752804704448756
  step_profile.sample_ms_total = 127.5953599512577
  step_profile.step_ms_mean = 58.97230183955501
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.764705882352941
  step_profile.verify_fraction = 0.33002445511706546
  step_profile.verify_ms_mean = 19.46230178159826
  step_profile.verify_ms_total = 6617.182605743408
  host_syncs.by_kind = {'cuda': 39}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 35
  host_syncs.per_step = 1.1142857142857143
  host_syncs.per_target_forward = 1.1142857142857143
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 35
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 39
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
[hf_assisted]
  tokens_per_second_median = 20.25776424898342
  tokens_per_second_p25 = 19.350477197434962
  tokens_per_second_p75 = 22.45474429590343
  tokens_per_second_mean = 20.962728232167184
  gpu_memory_gb = 4.024886608123779
  forward_counts.drafter_forwards = 1565
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 6.037735849056604
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 222
  forward_counts.target_forwards_per_emitted_token = 0.1734375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 911}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 29.387096774193548
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 31
  host_syncs.total = 911
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 185
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 32
[hf_sample]
  tokens_per_second_median = 25.444714453071143
  tokens_per_second_p25 = 25.347879588597422
  tokens_per_second_p75 = 25.543325368357564
  tokens_per_second_mean = 25.460665407664525
  gpu_memory_gb = 3.9432334899902344
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_sample_compiled]
  tokens_per_second_median = 49.24821559481592
  tokens_per_second_p25 = 49.19260891088411
  tokens_per_second_p75 = 49.33891386116446
  tokens_per_second_mean = 49.25829427978938
  gpu_memory_gb = 3.936194896697998
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = HumanEval/23
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = HumanEval/23
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile

==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=mt_bench  mode=greedy
commit=8d036d7  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 19.572806929034726
  tokens_per_second_p25 = 18.512325739597685
  tokens_per_second_p75 = 20.655499375866352
  tokens_per_second_mean = 19.853394600744853
  acceptance_rate_mean = 0.5868378012374362
  gpu_memory_gb = 3.902125835418701
  speedup_vs_flashspec_compiled_mean = 0.3503103364092562
  speedup_vs_flashspec_compiled_median = 0.3433727373539831
  speedup_vs_flashspec_compiled_std = 0.025886261084878966
  speedup_vs_hf_assisted_mean = 1.1212079242286672
  speedup_vs_hf_assisted_median = 1.1001251488328485
  speedup_vs_hf_assisted_std = 0.22263493260444342
  speedup_vs_hf_greedy_compiled_mean = 0.39676206169924877
  speedup_vs_hf_greedy_compiled_median = 0.3925474950495035
  speedup_vs_hf_greedy_compiled_std = 0.06025238708041143
  speedup_vs_hf_greedy_mean = 0.7885103743011131
  speedup_vs_hf_greedy_median = 0.7776848037196118
  speedup_vs_hf_greedy_std = 0.12246552766384723
  forward_counts.drafter_forwards = 1578
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.2653061224489797
  forward_counts.n_generations = 10
  forward_counts.n_steps = 392
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 402
  forward_counts.target_forwards_per_emitted_token = 0.3140625
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0026294115721332456
  step_profile.bookkeeping_ms_mean = 0.44171877688139066
  step_profile.bookkeeping_ms_total = 173.15376053750515
  step_profile.draft_fraction = 0.7735828371957351
  step_profile.draft_ms_mean = 129.95533612309669
  step_profile.draft_ms_total = 50942.491760253906
  step_profile.n_steps = 392.0
  step_profile.sample_fraction = 0.0007980146525395403
  step_profile.sample_ms_mean = 0.13405967327024865
  step_profile.sample_ms_total = 52.551391921937466
  step_profile.step_ms_mean = 167.99149344392046
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.2653061224489797
  step_profile.verify_fraction = 0.2229897365795922
  step_profile.verify_ms_mean = 37.46037887067211
  step_profile.verify_ms_total = 14684.468517303467
  host_syncs.by_kind = {'cuda': 47}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 44
  host_syncs.per_step = 1.0681818181818181
  host_syncs.per_target_forward = 1.0681818181818181
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 44
  host_syncs.total = 47
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 177
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 45
  greedy_token_match.first_divergence = [None, 44, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 9
  greedy_token_match.reference = hf_greedy
[flashspec_compiled]
  tokens_per_second_median = 55.69633073174627
  tokens_per_second_p25 = 52.71416447738182
  tokens_per_second_p75 = 57.00448545307138
  tokens_per_second_mean = 56.80027231478063
  acceptance_rate_mean = 0.5310738087770345
  gpu_memory_gb = 3.9118690490722656
  speedup_vs_flashspec_ucb_mean = 2.867865998607462
  speedup_vs_flashspec_ucb_median = 2.912287255033106
  speedup_vs_flashspec_ucb_std = 0.17958794438404146
  speedup_vs_hf_assisted_mean = 3.22313076066166
  speedup_vs_hf_assisted_median = 3.0887126152344058
  speedup_vs_hf_assisted_std = 0.7313143295058757
  speedup_vs_hf_greedy_compiled_mean = 1.1352602935236473
  speedup_vs_hf_greedy_compiled_median = 1.1161741322307486
  speedup_vs_hf_greedy_compiled_std = 0.17316374994830058
  speedup_vs_hf_greedy_mean = 2.255934534214634
  speedup_vs_hf_greedy_median = 2.2281081944682457
  speedup_vs_hf_greedy_std = 0.3504763225478148
  forward_counts.drafter_forwards = 1618
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.18407960199005
  forward_counts.n_generations = 10
  forward_counts.n_steps = 402
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 412
  forward_counts.target_forwards_per_emitted_token = 0.321875
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.008184116014510088
  step_profile.bookkeeping_ms_mean = 0.46802674586055293
  step_profile.bookkeeping_ms_total = 188.14675183594227
  step_profile.draft_fraction = 0.6533576809614974
  step_profile.draft_ms_mean = 37.36370168277873
  step_profile.draft_ms_total = 15020.20807647705
  step_profile.n_steps = 402.0
  step_profile.sample_fraction = 0.0025528041332819585
  step_profile.sample_ms_mean = 0.1459877412784752
  step_profile.sample_ms_total = 58.68707199394703
  step_profile.step_ms_mean = 57.18720812739721
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.18407960199005
  step_profile.verify_fraction = 0.33590539889071064
  step_profile.verify_ms_mean = 19.209491957479447
  step_profile.verify_ms_total = 7722.215766906738
  host_syncs.by_kind = {'cuda': 48}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 44
  host_syncs.per_step = 1.0909090909090908
  host_syncs.per_target_forward = 1.0909090909090908
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 44
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 48
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
  greedy_token_match_compiled.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match_compiled.n_prompts = 10
  greedy_token_match_compiled.prompts_matched = 10
  greedy_token_match_compiled.reference = hf_greedy_compiled
[hf_assisted]
  tokens_per_second_median = 19.849014490485814
  tokens_per_second_p25 = 15.136578031686579
  tokens_per_second_p75 = 22.22945913877053
  tokens_per_second_mean = 18.491069158789372
  gpu_memory_gb = 3.9825186729431152
  forward_counts.drafter_forwards = 1803
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.8905775075987843
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 339
  forward_counts.target_forwards_per_emitted_token = 0.26484375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 1184}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 28.878048780487806
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 41
  host_syncs.total = 1184
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 310
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 42
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy
[hf_greedy]
  tokens_per_second_median = 25.20267917967235
  tokens_per_second_p25 = 25.112631097015935
  tokens_per_second_p75 = 25.30654132302575
  tokens_per_second_mean = 25.1926794648301
  gpu_memory_gb = 3.9123334884643555
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_greedy_compiled]
  tokens_per_second_median = 49.99045416467851
  tokens_per_second_p25 = 49.59376567557413
  tokens_per_second_p75 = 50.24789576812648
  tokens_per_second_mean = 50.037195290721534
  gpu_memory_gb = 3.9069666862487793
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
  greedy_token_match.first_divergence = [None, None, None, None, None, None, None, None, None, None]
  greedy_token_match.n_prompts = 10
  greedy_token_match.prompts_matched = 10
  greedy_token_match.reference = hf_greedy

==============================================================================
target=Qwen/Qwen2.5-1.5B-Instruct  drafter=Qwen/Qwen2.5-0.5B-Instruct  dataset=mt_bench  mode=sample T=0.7
commit=8d036d7  n_samples=10
------------------------------------------------------------------------------
[flashspec_ucb]
  tokens_per_second_median = 20.18121040233048
  tokens_per_second_p25 = 18.115274461600237
  tokens_per_second_p75 = 21.17643975210253
  tokens_per_second_mean = 20.096723582882642
  acceptance_rate_mean = 0.5437414602239845
  gpu_memory_gb = 3.9191641807556152
  speedup_vs_flashspec_compiled_mean = 0.3764460647137391
  speedup_vs_flashspec_compiled_median = 0.3495936664091053
  speedup_vs_flashspec_compiled_std = 0.07162724933613403
  speedup_vs_hf_assisted_mean = 1.5573400316920538
  speedup_vs_hf_assisted_median = 1.234533396317398
  speedup_vs_hf_assisted_std = 0.6351021601650224
  speedup_vs_hf_sample_compiled_mean = 0.40861688458872336
  speedup_vs_hf_sample_compiled_median = 0.41020068096677864
  speedup_vs_hf_sample_compiled_std = 0.05322998941432055
  speedup_vs_hf_sample_mean = 0.7957119628167473
  speedup_vs_hf_sample_median = 0.7952682871520586
  speedup_vs_hf_sample_std = 0.10406410953516466
  forward_counts.drafter_forwards = 1546
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.3333333333333335
  forward_counts.n_generations = 10
  forward_counts.n_steps = 384
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 394
  forward_counts.target_forwards_per_emitted_token = 0.3078125
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.002556766829163343
  step_profile.bookkeeping_ms_mean = 0.43055691667056334
  step_profile.bookkeeping_ms_total = 165.33385600149632
  step_profile.draft_fraction = 0.7712902676519304
  step_profile.draft_ms_mean = 129.88449150323868
  step_profile.draft_ms_total = 49875.64473724365
  step_profile.n_steps = 384.0
  step_profile.sample_fraction = 0.0022661875676687343
  step_profile.sample_ms_mean = 0.3816236665006727
  step_profile.sample_ms_total = 146.54348793625832
  step_profile.step_ms_mean = 168.39897630064903
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.3333333333333335
  step_profile.verify_fraction = 0.22388677795123751
  step_profile.verify_ms_mean = 37.70230421423912
  step_profile.verify_ms_total = 14477.684818267822
  host_syncs.by_kind = {'cuda': 48}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 46
  host_syncs.per_step = 1.0434782608695652
  host_syncs.per_target_forward = 1.0434782608695652
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 46
  host_syncs.total = 48
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 185
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 47
[flashspec_compiled]
  tokens_per_second_median = 53.447483345537975
  tokens_per_second_p25 = 50.30655289847409
  tokens_per_second_p75 = 59.43461086804307
  tokens_per_second_mean = 54.52993649550763
  acceptance_rate_mean = 0.5374968568894605
  gpu_memory_gb = 3.923170566558838
  speedup_vs_flashspec_ucb_mean = 2.7251661977492136
  speedup_vs_flashspec_ucb_median = 2.8604796314571175
  speedup_vs_flashspec_ucb_std = 0.36378084390790744
  speedup_vs_hf_assisted_mean = 4.188705985115218
  speedup_vs_hf_assisted_median = 3.3014455192389267
  speedup_vs_hf_assisted_std = 1.7423536010992045
  speedup_vs_hf_sample_compiled_mean = 1.1085546809598585
  speedup_vs_hf_sample_compiled_median = 1.0837177249808096
  speedup_vs_hf_sample_compiled_std = 0.18760202573004073
  speedup_vs_hf_sample_mean = 2.159467717159328
  speedup_vs_hf_sample_median = 2.119677479254399
  speedup_vs_hf_sample_std = 0.3698479185375882
  forward_counts.drafter_forwards = 1562
  forward_counts.drafter_forwards_per_step = 4.0
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.2989690721649483
  forward_counts.n_generations = 10
  forward_counts.n_steps = 388
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 398
  forward_counts.target_forwards_per_emitted_token = 0.3109375
  forward_counts.target_forwards_per_step = 1.0
  forward_counts.tokens_emitted = 1280
  step_profile.bookkeeping_fraction = 0.0072542078509674505
  step_profile.bookkeeping_ms_mean = 0.4521320401162831
  step_profile.bookkeeping_ms_total = 175.42723156511784
  step_profile.draft_fraction = 0.6751031762793371
  step_profile.draft_ms_mean = 42.07706515322027
  step_profile.draft_ms_total = 16325.901279449463
  step_profile.n_steps = 388.0
  step_profile.sample_fraction = 0.006065551992674849
  step_profile.sample_ms_mean = 0.37804684580601366
  step_profile.sample_ms_total = 146.6821761727333
  step_profile.step_ms_mean = 62.32686592457989
  step_profile.tokens_emitted = 1280.0
  step_profile.tokens_per_step_mean = 3.2989690721649483
  step_profile.verify_fraction = 0.3115770638770206
  step_profile.verify_ms_mean = 19.419621885437326
  step_profile.verify_ms_total = 7534.813291549683
  host_syncs.by_kind = {'cuda': 50}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.n_steps = 46
  host_syncs.per_step = 1.0869565217391304
  host_syncs.per_target_forward = 1.0869565217391304
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 46
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 50
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile
[hf_assisted]
  tokens_per_second_median = 15.343815220295882
  tokens_per_second_p25 = 9.802755592886559
  tokens_per_second_p75 = 19.533044949645106
  tokens_per_second_mean = 15.119661278180597
  gpu_memory_gb = 4.005769729614258
  forward_counts.drafter_forwards = 2675
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 3.9751552795031055
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 332
  forward_counts.target_forwards_per_emitted_token = 0.259375
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 1603}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 42.18421052631579
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 38
  host_syncs.total = 1603
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 388
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 39
[hf_sample]
  tokens_per_second_median = 25.344508384460696
  tokens_per_second_p25 = 25.114667710180996
  tokens_per_second_p75 = 25.389395380475488
  tokens_per_second_mean = 25.261964877739068
  gpu_memory_gb = 3.9262752532958984
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = measured with a forward hook
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 130}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0236220472440944
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.total = 130
  nonfinite_logits.drafter_entries = 0
  nonfinite_logits.drafter_forwards_checked = 0
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = the untimed probe generation on the first prompt
  nonfinite_logits.target_entries = 0
  nonfinite_logits.target_forwards_checked = 128
[hf_sample_compiled]
  tokens_per_second_median = 49.189297069159096
  tokens_per_second_p25 = 49.090311507338804
  tokens_per_second_p75 = 49.27322962156904
  tokens_per_second_mean = 49.18980329375679
  gpu_memory_gb = 3.9209084510803223
  forward_counts.emitted_tokens_per_target_forward_after_prefill = 1.0078740157480315
  forward_counts.n_generations = 10
  forward_counts.source = derived: a hook would force the compiled forward to recompile
  forward_counts.target_forwards = 1280
  forward_counts.target_forwards_per_emitted_token = 1.0
  forward_counts.tokens_emitted = 1280
  host_syncs.by_kind = {'cuda': 132}
  host_syncs.counter = cuda_sync_debug_mode
  host_syncs.per_target_forward = 1.0393700787401574
  host_syncs.prompt_id = 81
  host_syncs.target_forwards_after_prefill = 127
  host_syncs.target_forwards_source = derived: one per step / per decoded token
  host_syncs.total = 132
  nonfinite_logits.prompt_id = 81
  nonfinite_logits.scope = not checked: a forward hook would force the compiled forward to recompile

