Operator/Benchmark Sweep

Run the benchmark suite with multiple operator presets and compare objective/runtime metrics.

[1]:

import sys
from pathlib import Path

PROJECT_ROOT = Path.cwd().resolve()
while PROJECT_ROOT != PROJECT_ROOT.parent and not (PROJECT_ROOT / "pyproject.toml").exists():
    PROJECT_ROOT = PROJECT_ROOT.parent
if not (PROJECT_ROOT / "pyproject.toml").exists():
    raise RuntimeError(
        "Notebook must be executed within a FHOPS checkout (pyproject.toml not found)."
    )
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))


from fhops.cli.benchmarks import run_benchmark_suite

BENCH_DIR = PROJECT_ROOT / "docs/examples/analytics/data/operator_sweep"
summary = run_benchmark_suite(
    [PROJECT_ROOT / "examples/synthetic/medium/scenario.yaml"],
    BENCH_DIR,
    time_limit=5,
    sa_iters=150,
    include_mip=False,
    preset_comparisons=["explore", "stabilise"],
)
summary

                         FHOPS Benchmark Summary                          
┏━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━┓
┃ scenario ┃ solver ┃ preset_label ┃ objective ┃ runtime_s ┃ assignments ┃
┡━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━┩
│ user-1   │ sa     │ default      │ 89.370    │ 0.04      │ 9           │
│ user-1   │ sa     │ explore      │ 94.371    │ 0.09      │ 9           │
│ user-1   │ sa     │ stabilise    │ 94.908    │ 0.08      │ 9           │
└──────────┴────────┴──────────────┴───────────┴───────────┴─────────────┘

[1]:

	scenario	scenario_path	solver	objective	runtime_s	assignments	kpi_total_production	kpi_completed_blocks	kpi_utilisation_ratio_mean_shift	kpi_utilisation_ratio_weighted_shift	...	operators_stats	scenario_key	objective_vs_mip_gap	objective_vs_mip_ratio	solver_category	best_heuristic_solver	best_heuristic_objective	best_heuristic_runtime_s	objective_gap_vs_best_heuristic	runtime_ratio_vs_best_heuristic
0	user-1	/home/gep/projects/fhops/examples/synthetic/me...	sa	89.370	0.043104	9	89.370	5.0	1.0	1.0	...	{"move": {"acceptance_rate": 1.0, "accepted": ...	user-1	<NA>	<NA>	heuristic	sa	89.37	0.043104	0.000	1.000000
1	user-1	/home/gep/projects/fhops/examples/synthetic/me...	sa	94.371	0.093004	9	94.371	5.0	1.0	1.0	...	{"block_insertion": {"acceptance_rate": 1.0, "...	user-1	<NA>	<NA>	heuristic	sa	89.37	0.043104	5.001	2.157648
2	user-1	/home/gep/projects/fhops/examples/synthetic/me...	sa	94.908	0.075353	9	94.908	6.0	1.0	1.0	...	{"block_insertion": {"acceptance_rate": 1.0, "...	user-1	<NA>	<NA>	heuristic	sa	89.37	0.043104	5.538	1.748160

3 rows × 35 columns

Objective Comparison

[2]:

objective_pivot = summary.pivot_table(index="preset_label", values="objective")
objective_pivot

[2]:

	objective
preset_label
default	89.370
explore	94.371
stabilise	94.908

Runtime Comparison

[3]:

runtime_pivot = summary.pivot_table(index="preset_label", values="runtime_s")
runtime_pivot

[3]:

	runtime_s
preset_label
default	0.043104
explore	0.093004
stabilise	0.075353