= 10
MAX_TIME = 10
INIT_SIZE ="10-river"
PREFIX= .1 K
11 River Hyperparameter Tuning with SPOT HATR
11.1 Step 1: Setup
Before we consider the detailed experimental setup, we select the parameters that affect run time, initial design size and the device that is used.
Caution: Run time and initial design size should be increased for real experiments
- MAX_TIME is set to one minute for demonstration purposes. For real experiments, this should be increased to at least 1 hour.
- INIT_SIZE is set to 5 for demonstration purposes. For real experiments, this should be increased to at least 10.
- K is set to 0.1 for demonstration purposes. For real experiments, this should be increased to at least 1.
12 Chapter 10: Sequential Parameter Optimization
12.1 river
Hyperparameter Tuning: HATR with Friedman Drift Data
- This notebook exemplifies hyperparameter tuning with SPOT (spotPython and spotRiver).
- The hyperparameter software SPOT was developed in R (statistical programming language), see Open Access book “Hyperparameter Tuning for Machine and Deep Learning with R - A Practical Guide”, available here: https://link.springer.com/book/10.1007/978-981-19-5170-1.
- This notebook demonstrates hyperparameter tuning for
river
. It is based on the notebook “Incremental decision trees in river: the Hoeffding Tree case”, see: https://riverml.xyz/0.15.0/recipes/on-hoeffding-trees/#42-regression-tree-splitters. - Here we will use the river
HTR
andHATR
functions as in “Incremental decision trees in river: the Hoeffding Tree case”, see: https://riverml.xyz/0.15.0/recipes/on-hoeffding-trees/#42-regression-tree-splitters.
12.2 Example 1: HATR Hyperparameter
12.3 0. Initialization of the Empty fun_control
Dictionary
from spotPython.utils.init import fun_control_init
= fun_control_init(tensorboard_path=f"./runs/{experiment_name}/") fun_control
12.4 1. Load Data: The Friedman Drift Data
= 7*24
horizon = K
k = int(k*100_000)
n_total = n_total
n_samples = int(k*25_000)
p_1 = int(k*50_000)
p_2 =(p_1, p_2)
position= 1_000
n_train = n_train + p_1 - 12
a = a + 12 b
- Since we also need a
river
version of the data below for plotting the model, the corresponding data set is generated here. Note:spotRiver
uses thetrain
andtest
data sets, whileriver
uses theX
andy
data sets
from river.datasets import synth
import pandas as pd
= synth.FriedmanDrift(
dataset ='gra',
drift_type=position,
position=123
seed )
from spotRiver.utils.data_conversion import convert_to_df
= "y"
target_column = convert_to_df(dataset, target_column=target_column, n_total=n_total) df
# Add column names x1 until x10 to the first 10 columns of the dataframe and the column name y to the last column
= [f"x{i}" for i in range(1, 11)] + ["y"]
df.columns
= df[:n_train]
train = df[n_train:]
test #
"data": None, # dataset,
fun_control.update({"train": train,
"test": test,
"n_samples": n_samples,
"target_column": target_column})
12.5 2. Specification of the Preprocessing Model
from river import preprocessing
= preprocessing.StandardScaler()
prep_model "prep_model": prep_model}) fun_control.update({
12.6 3. Select algorithm
and core_model_hyper_dict
- The
river
model (HATR
) is selected. - Furthermore, the corresponding hyperparameters, see: https://riverml.xyz/0.15.0/api/tree/HoeffdingTreeRegressor/ are selected (incl. type information, names, and bounds).
- The corresponding hyperparameter dictionary is added to the
fun_control
dictionary. - Alternatively, you can load a local hyper_dict. Simply set
river_hyper_dict.json
as the filename. Iffilename
is set toNone
, the hyper_dict is loaded from thespotRiver
package.
from river.tree import HoeffdingAdaptiveTreeRegressor
from spotRiver.data.river_hyper_dict import RiverHyperDict
from spotPython.hyperparameters.values import add_core_model_to_fun_control
= HoeffdingAdaptiveTreeRegressor
core_model =core_model,
add_core_model_to_fun_control(core_model=fun_control,
fun_control=RiverHyperDict,
hyper_dict=None) filename
12.7 4. Modify hyper_dict
Hyperparameters for the Selected Algorithm aka core_model
12.7.1 Modify hyperparameter of type factor
# modify_hyper_parameter_levels(fun_control, "leaf_model", ["LinearRegression"])
# fun_control["core_model_hyper_dict"]
12.7.2 Modify hyperparameter of type numeric and integer (boolean)
from spotPython.hyperparameters.values import modify_hyper_parameter_bounds
"delta", bounds=[1e-10, 1e-6])
modify_hyper_parameter_bounds(fun_control, # modify_hyper_parameter_bounds(fun_control, "min_samples_split", bounds=[3, 20])
"merit_preprune", [0, 0]) modify_hyper_parameter_bounds(fun_control,
12.8 5. Selection of the Objective (Loss) Function
There are two metrics:
1. `metric` is used for the river based evaluation via `eval_oml_iter_progressive`.
2. `metric_sklearn` is used for the sklearn based evaluation via `eval_oml_horizon`.
import numpy as np
from river import metrics
from sklearn.metrics import mean_absolute_error
= np.array([1, 1/1000, 1/1000])*10_000.0
weights = 7*24
horizon = 2
oml_grace_period = 100
step = 1.0
weight_coeff
fun_control.update({"horizon": horizon,
"oml_grace_period": oml_grace_period,
"weights": weights,
"step": step,
"log_level": 50,
"weight_coeff": weight_coeff,
"metric": metrics.MAE(),
"metric_sklearn": mean_absolute_error
})
12.9 6. Calling the SPOT Function
12.9.1 Prepare the SPOT Parameters
- Get types and variable names as well as lower and upper bounds for the hyperparameters.
from spotPython.hyperparameters.values import (
get_var_type,
get_var_name,
get_bound_values
)= get_var_type(fun_control)
var_type = get_var_name(fun_control)
var_name = get_bound_values(fun_control, "lower")
lower = get_bound_values(fun_control, "upper") upper
from spotPython.utils.eda import gen_design_table
print(gen_design_table(fun_control))
12.9.2 Run the Spot
Optimizer
from spotRiver.fun.hyperriver import HyperRiver
= HyperRiver().fun_oml_horizon fun
from spotPython.hyperparameters.values import get_default_hyperparameters_as_array
= get_default_hyperparameters_as_array(fun_control) X_start
- Run SPOT for approx. x mins (
max_time
). - Note: the run takes longer, because the evaluation time of initial design (here:
init_size
= INIT_SIZE as specified above) is not considered.
from spotPython.spot import spot
from math import inf
import numpy as np
= spot.Spot(fun=fun,
spot_tuner = lower,
lower = upper,
upper = inf,
fun_evals = MAX_TIME,
max_time = np.sqrt(np.spacing(1)),
tolerance_x = var_type,
var_type = var_name,
var_name = True,
show_progress= fun_control,
fun_control ={"init_size": INIT_SIZE},
design_control={"noise": True,
surrogate_control"cod_type": "norm",
"min_theta": -4,
"max_theta": 3,
"n_theta": len(var_name),
"model_fun_evals": 10_000})
=X_start) spot_tuner.run(X_start
12.9.3 4 Results
from spotPython.utils.file import save_pickle
save_pickle(spot_tuner, experiment_name)
from spotPython.utils.file import load_pickle
= load_pickle(experiment_name) spot_tuner
- Show the Progress of the hyperparameter tuning:
=True, filename="./figures/" + experiment_name+"_progress.pdf") spot_tuner.plot_progress(log_y
- Print the Results
print(gen_design_table(fun_control=fun_control, spot=spot_tuner))
12.10 Show variable importance
=0.0025, filename="./figures/" + experiment_name+"_importance.pdf") spot_tuner.plot_importance(threshold
12.11 Build and Evaluate HTR Model with Tuned Hyperparameters
= test.shape[0]
m = int(m/2)-50
a = int(m/2) b
13 Der große Datensatz
Caution: Increased Friedman-Drift Data Set
- The Friedman-Drift Data Set is increased by a factor of two to show the transferability of the hyperparameter tuning results.
- Larger values of
k
lead to a longer run time.
= 7*24
horizon = 0.2
k = int(k*100_000)
n_total = n_total
n_samples = int(k*25_000)
p_1 = int(k*50_000)
p_2 =(p_1, p_2)
position= 1_000
n_train = n_train + p_1 - 12
a = a + 12 b
from river.datasets import synth
= synth.FriedmanDrift(
dataset ='gra',
drift_type=position,
position=123
seed )
from spotRiver.utils.data_conversion import convert_to_df
= "y"
target_column = convert_to_df(dataset, target_column=target_column, n_total=n_total)
df # Add column names x1 until x10 to the first 10 columns of the dataframe and the column name y to the last column
= [f"x{i}" for i in range(1, 11)] + ["y"]
df.columns = df[:n_train]
train = df[n_train:]
test = "y"
target_column #
"data": None, # dataset,
fun_control.update({"train": train,
"test": test,
"n_samples": n_samples,
"target_column": target_column})
13.1 Get Default Hyperparameters
# fun_control was modified, we generate a new one with the original
# default hyperparameters
from spotPython.hyperparameters.values import get_one_core_model_from_X
from spotPython.hyperparameters.values import get_default_hyperparameters_as_array
= get_default_hyperparameters_as_array(fun_control)
X_start = get_one_core_model_from_X(X_start, fun_control)
model_default model_default
from spotRiver.evaluation.eval_bml import eval_oml_horizon
= eval_oml_horizon(
df_eval_default, df_true_default =model_default,
model=fun_control["train"],
train=fun_control["test"],
test=fun_control["target_column"],
target_column=fun_control["horizon"],
horizon=fun_control["oml_grace_period"],
oml_grace_period=fun_control["metric_sklearn"],
metric )
from spotRiver.evaluation.eval_bml import plot_bml_oml_horizon_metrics, plot_bml_oml_horizon_predictions
=["default"]
df_labels= [df_eval_default], log_y=False, df_labels=df_labels, metric=fun_control["metric_sklearn"])
plot_bml_oml_horizon_metrics(df_eval = [df_true_default[a:b]], target_column=target_column, df_labels=df_labels) plot_bml_oml_horizon_predictions(df_true
13.2 Get SPOT Results
from spotPython.hyperparameters.values import get_one_core_model_from_X
= spot_tuner.to_all_dim(spot_tuner.min_X.reshape(1,-1))
X = get_one_core_model_from_X(X, fun_control)
model_spot model_spot
= eval_oml_horizon(
df_eval_spot, df_true_spot =model_spot,
model=fun_control["train"],
train=fun_control["test"],
test=fun_control["target_column"],
target_column=fun_control["horizon"],
horizon=fun_control["oml_grace_period"],
oml_grace_period=fun_control["metric_sklearn"],
metric )
=["default", "spot"]
df_labels= [df_eval_default, df_eval_spot], log_y=False, df_labels=df_labels, metric=fun_control["metric_sklearn"], filename="./figures/" + experiment_name+"_metrics.pdf") plot_bml_oml_horizon_metrics(df_eval
= int(m/2)+20
a = int(m/2)+50
b = [df_true_default[a:b], df_true_spot[a:b]], target_column=target_column, df_labels=df_labels, filename="./figures/" + experiment_name+"_predictions.pdf") plot_bml_oml_horizon_predictions(df_true
from spotPython.plot.validation import plot_actual_vs_predicted
=df_true_default["y"], y_pred=df_true_default["Prediction"], title="Default")
plot_actual_vs_predicted(y_test=df_true_spot["y"], y_pred=df_true_spot["Prediction"], title="SPOT") plot_actual_vs_predicted(y_test
13.3 Visualize Regression Trees
= dataset.take(n_total)
dataset_f for x, y in dataset_f:
model_default.learn_one(x, y)
Caution: Large Trees
- Since the trees are large, the visualization is suppressed by default.
- To visualize the trees, uncomment the following line.
# model_default.draw()
model_default.summary
13.3.1 Spot Model
= dataset.take(n_total)
dataset_f for x, y in dataset_f:
model_spot.learn_one(x, y)
Caution: Large Trees
- Since the trees are large, the visualization is suppressed by default.
- To visualize the trees, uncomment the following line.
# model_spot.draw()
model_spot.summary
from spotPython.utils.eda import compare_two_tree_models
print(compare_two_tree_models(model_default, model_spot))
13.4 Detailed Hyperparameter Plots
= "./figures/" + experiment_name
filename =filename) spot_tuner.plot_important_hyperparameter_contour(filename
13.5 Parallel Coordinates Plots
spot_tuner.parallel_plot()
13.6 Plot all Combinations of Hyperparameters
- Warning: this may take a while.
= False
PLOT_ALL if PLOT_ALL:
= spot_tuner.k
n for i in range(n-1):
for j in range(i+1, n):
=i, j=j, min_z=min_z, max_z = max_z) spot_tuner.plot_contour(i