Source code for MED3pa.med3pa.experiment

"""
Orchestrates the execution of the med3pa method and integrates the functionality of other modules to run comprehensive experiments. 
It includes classes to manage and store results ``Med3paResults``, execute experiments like ``Med3paExperiment`` and ``Med3paDetectronExperiment``, and integrate results from the Detectron method ``Med3paDetectronResults``
"""
import json
import os
from typing import Any, Dict, List, Tuple, Type, Union

import numpy as np
from sklearn.model_selection import train_test_split

from MED3pa.datasets import DatasetsManager, MaskedDataset
from MED3pa.detectron.experiment import DetectronExperiment, DetectronResult, DetectronStrategy, EnhancedDisagreementStrategy
from MED3pa.med3pa.mdr import MDRCalculator
from MED3pa.med3pa.models import *
from MED3pa.med3pa.profiles import Profile, ProfilesManager
from MED3pa.med3pa.uncertainty import *
from MED3pa.models.base import BaseModelManager
from MED3pa.models.classification_metrics import *
from MED3pa.models.concrete_regressors import *


[docs]def to_serializable(obj: Any, additional_arg: Any = None) -> Any: """Convert an object to a JSON-serializable format. Args: obj (Any): The object to convert. Returns: Any: The JSON-serializable representation of the object. """ if isinstance(obj, np.ndarray): return obj.tolist() if isinstance(obj, (np.integer, np.floating)): return obj.item() if isinstance(obj, Profile): if additional_arg is not None: return obj.to_dict(additional_arg) else: return obj.to_dict() if isinstance(obj, dict): return {k: to_serializable(v) for k, v in obj.items()} if isinstance(obj, list): return [to_serializable(v) for v in obj] return obj
[docs]class Med3paResults: """ Class to store and manage results from the MED3PA method. """ def __init__(self) -> None: self.metrics_by_dr: Dict[int, Dict] = {} self.models_evaluation: Dict[str, Dict] = {} self.profiles_manager: ProfilesManager = None self.datasets: Dict[int, MaskedDataset] = {}
[docs] def set_metrics_by_dr(self, metrics_by_dr: Dict) -> None: """ Set the calculated metrics by declaration rate. Args: metrics_by_dr (Dict): Dictionary of metrics by declaration rate. """ self.metrics_by_dr = metrics_by_dr
[docs] def set_profiles_manager(self, profile_manager : ProfilesManager) -> None: """ Set the profile manager for this Med3paResults instance. Args: profile_manager (ProfilesManager): The ProfileManager instance. """ self.profiles_manager = profile_manager
[docs] def set_models_evaluation(self, ipc_evaluation: Dict, apc_evaluation: Dict=None) -> None: """ Set models evaluation metrics. Args: ipc_evaluation (Dict): Evaluation metrics for IPC model. apc_evaluation (Dict): Evaluation metrics for APC model. """ self.models_evaluation['IPC_evaluation'] = ipc_evaluation if apc_evaluation is not None: self.models_evaluation['APC_evaluation'] = apc_evaluation
[docs] def set_dataset(self, samples_ratio: int, dataset: MaskedDataset) -> None: """ Saves the dataset for a given sample ratio. Args: samples_ratio (int): The sample ratio. dataset (MaskedDataset): The MaskedDataset instance. """ self.datasets[samples_ratio] = dataset
[docs] def save(self, file_path: str) -> None: """ Saves the experiment results. Args: file_path (str): The file path to save the JSON files. """ # Ensure the main directory exists os.makedirs(file_path, exist_ok=True) with open(f'{file_path}/metrics_dr.json', 'w') as file: json.dump(self.metrics_by_dr, file, default=to_serializable, indent=4) if self.profiles_manager is not None: with open(f'{file_path}/profiles.json', 'w') as file: json.dump(self.profiles_manager.get_profiles(), file, default=to_serializable, indent=4) with open(f'{file_path}/lost_profiles.json', 'w') as file: json.dump(self.profiles_manager.get_lost_profiles(), file, default=lambda x: to_serializable(x, additional_arg = False), indent=4) if self.models_evaluation is not None: with open(f'{file_path}/models_evaluation.json', 'w') as file: json.dump(self.models_evaluation, file, default=to_serializable, indent=4) for samples_ratio, dataset in self.datasets.items(): dataset_path = os.path.join(file_path, f'dataset_{samples_ratio}.csv') dataset.save_to_csv(dataset_path)
[docs] def get_profiles_manager(self) -> ProfilesManager: """ Retrieves the profiles manager for this Med3paResults instance """ return self.profiles_manager
[docs]class Med3paExperiment: """ Class to run the MED3PA method experiment. """
[docs] @staticmethod def run(datasets_manager: DatasetsManager, base_model_manager: BaseModelManager = None, uncertainty_metric: Type[UncertaintyMetric] = AbsoluteError, ipc_type: str = 'RandomForestRegressor', ipc_params: Dict = None, ipc_grid_params: Dict = None, ipc_cv: int = 4, apc_params: Dict = None, apc_grid_params: Dict = None, apc_cv: int = 4, samples_ratio_min: int = 0, samples_ratio_max: int = 50, samples_ratio_step: int = 5, med3pa_metrics: List[str] = [], evaluate_models: bool = False, mode: str = 'mpc', models_metrics: List[str] = ['MSE', 'RMSE']) -> Tuple[Med3paResults, Med3paResults]: """Runs the MED3PA experiment on both reference and testing sets. Args: datasets_manager (DatasetsManager): the datasets manager containing the dataset to use in the experiment. base_model_manager (BaseModelManager, optional): Instance of BaseModelManager to get the base model, by default None. uncertainty_metric (Type[UncertaintyMetric], optional): Instance of UncertaintyMetric to calculate uncertainty, by default AbsoluteError. ipc_type (str, optional): The regressor model to use for IPC, by default RandomForestRegressor. ipc_params (dict, optional): Parameters for initializing the IPC regressor model, by default None. ipc_grid_params (dict, optional): Grid search parameters for optimizing the IPC model, by default None. ipc_cv (int, optional): Number of cross-validation folds for optimizing the IPC model, by default None. apc_params (dict, optional): Parameters for initializing the APC regressor model, by default None. apc_grid_params (dict, optional): Grid search parameters for optimizing the APC model, by default None. apc_cv (int, optional): Number of cross-validation folds for optimizing the APC model, by default None. samples_ratio_min (int, optional): Minimum sample ratio, by default 0. samples_ratio_max (int, optional): Maximum sample ratio, by default 50. samples_ratio_step (int, optional): Step size for sample ratio, by default 5. med3pa_metrics (list of str, optional): List of metrics to calculate, by default ['Auc', 'Accuracy', 'BalancedAccuracy']. evaluate_models (bool, optional): Whether to evaluate the models, by default False. models_metrics (list of str, optional): List of metrics for model evaluation, by default ['MSE', 'RMSE']. Returns: Tuple[Med3paResults, Med3paResults]: the results of the MED3PA experiment on the reference set and testing set. """ print("Running MED3pa Experiment on the reference set:") results_reference = Med3paExperiment._run_by_set(datasets_manager=datasets_manager,set= 'reference',base_model_manager= base_model_manager, uncertainty_metric=uncertainty_metric, ipc_type=ipc_type, ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, apc_params=apc_params,apc_grid_params=apc_grid_params, apc_cv=apc_cv, samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step, med3pa_metrics=med3pa_metrics, evaluate_models=evaluate_models, models_metrics=models_metrics, mode=mode) print("Running MED3pa Experiment on the reference set:") results_testing = Med3paExperiment._run_by_set(datasets_manager=datasets_manager,set= 'testing',base_model_manager= base_model_manager, uncertainty_metric=uncertainty_metric, ipc_type=ipc_type, ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, apc_params=apc_params,apc_grid_params=apc_grid_params, apc_cv=apc_cv, samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step, med3pa_metrics=med3pa_metrics, evaluate_models=evaluate_models, models_metrics=models_metrics, mode=mode) return results_reference, results_testing
@staticmethod def _run_by_set(datasets_manager: DatasetsManager, set: str = 'reference', base_model_manager: BaseModelManager = None, uncertainty_metric: Type[UncertaintyMetric] = AbsoluteError, ipc_type: str = 'RandomForestRegressor', ipc_params: Dict = None, ipc_grid_params: Dict = None, ipc_cv: int = 4, apc_params: Dict = None, apc_grid_params: Dict = None, apc_cv: int = 4, samples_ratio_min: int = 0, samples_ratio_max: int = 50, samples_ratio_step: int = 5, med3pa_metrics: List[str] = [], evaluate_models: bool = False, mode: str = 'mpc', models_metrics: List[str] = ['MSE', 'RMSE']) -> Med3paResults: """Orchestrates the MED3PA experiment on one specific set of the dataset. Args: datasets_manager (DatasetsManager): the datasets manager containing the dataset to use in the experiment. base_model_manager (BaseModelManager, optional): Instance of BaseModelManager to get the base model, by default None. uncertainty_metric (Type[UncertaintyMetric], optional): Instance of UncertaintyMetric to calculate uncertainty, by default AbsoluteError. ipc_type (str, optional): The regressor model to use for IPC, by default RandomForestRegressor. ipc_params (dict, optional): Parameters for initializing the IPC regressor model, by default None. ipc_grid_params (dict, optional): Grid search parameters for optimizing the IPC model, by default None. ipc_cv (int, optional): Number of cross-validation folds for optimizing the IPC model, by default None. apc_params (dict, optional): Parameters for initializing the APC regressor model, by default None. apc_grid_params (dict, optional): Grid search parameters for optimizing the APC model, by default None. apc_cv (int, optional): Number of cross-validation folds for optimizing the APC model, by default None. samples_ratio_min (int, optional): Minimum sample ratio, by default 0. samples_ratio_max (int, optional): Maximum sample ratio, by default 50. samples_ratio_step (int, optional): Step size for sample ratio, by default 5. med3pa_metrics (list of str, optional): List of metrics to calculate, by default ['Auc', 'Accuracy', 'BalancedAccuracy']. evaluate_models (bool, optional): Whether to evaluate the models, by default False. models_metrics (list of str, optional): List of metrics for model evaluation, by default ['MSE', 'RMSE']. Returns: Med3paResults: the results of the MED3PA experiment. """ # retrieve the dataset based on the set type try: if set == 'reference': dataset = datasets_manager.get_dataset_by_type(dataset_type="reference", return_instance=True) elif set == 'testing': dataset = datasets_manager.get_dataset_by_type(dataset_type="testing", return_instance=True) else: raise ValueError("The set must be either the reference set or the testing set") except ValueError as e: dataset = None if dataset is None: return None valid_modes = ['mpc', 'apc', 'ipc'] if mode not in valid_modes: raise ValueError(f"Invalid mode '{mode}'. The mode must be one of {valid_modes}.") # retrieve different dataset components to calculate the metrics x = dataset.get_observations() y_true = dataset.get_true_labels() predicted_probabilities = dataset.get_pseudo_probabilities() features = datasets_manager.get_column_labels() # Initialize base model and predict probabilities if not provided if base_model_manager is None and predicted_probabilities is None: raise ValueError("Either the base model or the predicted probabilities should be provided!") if predicted_probabilities is None: base_model = base_model_manager.get_instance() predicted_probabilities = base_model.predict(x, True) dataset.set_pseudo_labels(predicted_probabilities) # Calculate uncertainty values uncertainty_calc = UncertaintyCalculator(uncertainty_metric) uncertainty_values = uncertainty_calc.calculate_uncertainty(x, predicted_probabilities, y_true) # set predicted labels dataset.set_pseudo_probs_labels(predicted_probabilities, 0.5) if evaluate_models: x_train, x_test, uncertainty_train, uncertainty_test = train_test_split(x, uncertainty_values, test_size=0.1, random_state=42) else: x_train = x uncertainty_train = uncertainty_values if med3pa_metrics == []: med3pa_metrics = ClassificationEvaluationMetrics.supported_metrics() results = Med3paResults() # Create and train IPCModel IPC_model = IPCModel(ipc_type, ipc_params) IPC_model.train(x_train, uncertainty_train) print("IPC Model training completed.") # optimize IPC model if grid params were provided if ipc_grid_params is not None: IPC_model.optimize(ipc_grid_params, ipc_cv, x_train, uncertainty_train) print("IPC Model optimization done.") # Predict IPC values IPC_values = IPC_model.predict(x) if mode in ['mpc', 'apc']: # Create and train APCModel APC_model = APCModel(features, apc_params) APC_model.train(x, IPC_values) print("APC Model training completed.") # optimize APC model if grid params were provided if apc_grid_params is not None: APC_model.optimize(apc_grid_params, apc_cv, x_train, uncertainty_train) print("APC Model optimization done.") profiles_manager = ProfilesManager(features) for samples_ratio in range(samples_ratio_min, samples_ratio_max + 1, samples_ratio_step): # Predict APC values APC_values = APC_model.predict(x, min_samples_ratio=samples_ratio) if mode == 'mpc': # Create and predict MPC values MPC_model = MPCModel(IPC_values=IPC_values, APC_values=APC_values) else: MPC_model = MPCModel(APC_values=APC_values) MPC_values = MPC_model.predict() dataset.set_confidence_scores(MPC_values) print("Confidence scores calculated for minimum_samples_ratio = ", samples_ratio) # Calculate profiles and their metrics by declaration rate tree = APC_model.treeRepresentation MDRCalculator.calc_profiles(profiles_manager, tree, MPC_values, samples_ratio) MDRCalculator.calc_metrics_by_profiles(profiles_manager, datasets_manager, med3pa_metrics, set=set) cloned_dataset = dataset.clone() results.set_profiles_manager(profiles_manager) results.set_dataset(samples_ratio=samples_ratio, dataset=cloned_dataset) print("Results extracted for minimum_samples_ratio = ", samples_ratio) # Calculate metrics by declaration rate # Create and predict MPC values using only the IPC values MPC_model = MPCModel(IPC_values=IPC_values) MPC_values = MPC_model.predict() dataset.set_confidence_scores(MPC_values) metrics_by_dr = MDRCalculator.calc_metrics_by_dr(datasets_manager=datasets_manager, metrics_list=med3pa_metrics, set=set) results.set_metrics_by_dr(metrics_by_dr) # evaluate models if evaluate_models: if mode in ['mpc', 'apc']: IPC_evaluation = IPC_model.evaluate(x_test, uncertainty_test, models_metrics) APC_evaluation = APC_model.evaluate(x_test, uncertainty_test, models_metrics) results.set_models_evaluation(IPC_evaluation, APC_evaluation) else : IPC_evaluation = IPC_model.evaluate(x_test, uncertainty_test, models_metrics) results.set_models_evaluation(IPC_evaluation, None) return results
[docs]class Med3paDetectronExperiment:
[docs] @staticmethod def run(datasets: DatasetsManager, base_model_manager: BaseModelManager, uncertainty_metric: Type[UncertaintyMetric] = AbsoluteError, training_params: Dict =None, samples_size: int = 20, samples_size_profiles: int = 10, ensemble_size: int = 10, num_calibration_runs: int = 100, patience: int = 3, test_strategies: Union[Type[DetectronStrategy], List[Type[DetectronStrategy]]] = EnhancedDisagreementStrategy, allow_margin: bool = False, margin: float = 0.05, ipc_type: str = 'RandomForestRegressor', ipc_params: Dict = None, ipc_grid_params: Dict = None, ipc_cv: int = None, apc_params: Dict = None, apc_grid_params: Dict = None, apc_cv: int = None, samples_ratio_min: int = 0, samples_ratio_max: int = 50, samples_ratio_step: int = 5, med3pa_metrics: List[str] = ['Auc', 'Accuracy', 'BalancedAccuracy'], evaluate_models: bool = False, models_metrics: List[str] = ['MSE', 'RMSE'], mode: str = 'mpc', all_dr: bool = False) -> Tuple[Med3paResults, Med3paResults, DetectronResult]: """Runs the MED3PA and Detectron experiment. Args: datasets (DatasetsManager): The datasets manager instance. training_params (dict): Parameters for training the models. base_model_manager (BaseModelManager): The base model manager instance. uncertainty_metric (Type[UncertaintyMetric]): The uncertainty metric to use. samples_size (int, optional): Sample size for the Detectron experiment, by default 20. samples_size_profiles (int, optional): Sample size for Profiles Detectron experiment, by default 10. ensemble_size (int, optional): Number of models in the ensemble, by default 10. num_calibration_runs (int, optional): Number of calibration runs, by default 100. patience (int, optional): Patience for early stopping, by default 3. test_strategies (Union[Type[DetectronStrategy], List[Type[DetectronStrategy]]]): strategies for testing disagreement, by default EnhancedDisagreementStrategy. allow_margin (bool, optional): Whether to allow a margin in the test, by default False. margin (float, optional): Margin value for the test, by default 0.05. ipc_type (str, optional): The regressor model to use for IPC, by default RandomForestRegressor. ipc_params (dict, optional): Parameters for initializing the IPC regressor model, by default None. ipc_grid_params (dict, optional): Grid search parameters for optimizing the IPC model, by default None. ipc_cv (int, optional): Number of cross-validation folds for optimizing the IPC model, by default None. apc_params (dict, optional): Parameters for initializing the APC regressor model, by default None. apc_grid_params (dict, optional): Grid search parameters for optimizing the APC model, by default None. apc_cv (int, optional): Number of cross-validation folds for optimizing the APC model, by default None. samples_ratio_min (int, optional): Minimum sample ratio, by default 0. samples_ratio_max (int, optional): Maximum sample ratio, by default 50. samples_ratio_step (int, optional): Step size for sample ratio, by default 5. med3pa_metrics (list of str, optional): List of metrics to calculate, by default ['Auc', 'Accuracy', 'BalancedAccuracy']. evaluate_models (bool, optional): Whether to evaluate the models, by default False. models_metrics (list of str, optional): List of metrics for model evaluation, by default ['MSE', 'RMSE']. all_dr (bool, optional): Whether to run for all declaration rates, by default False. Returns: Tuple[Med3paResults, Med3paResults, DetectronResult]: Results of MED3pa on reference and testing sets, plus Detectron Results. """ valid_modes = ['mpc', 'apc'] if mode not in valid_modes: raise ValueError(f"Invalid mode '{mode}'. The mode must be one of {valid_modes}.") reference_3pa_res, testing_3pa_res = Med3paExperiment.run(datasets_manager=datasets, base_model_manager=base_model_manager, uncertainty_metric=uncertainty_metric, ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, ipc_type=ipc_type, apc_params=apc_params, apc_grid_params=apc_grid_params, apc_cv=apc_cv, evaluate_models=evaluate_models, models_metrics=models_metrics, samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step, med3pa_metrics=med3pa_metrics, mode=mode) print("Running Global Detectron Experiment:") detectron_results = DetectronExperiment.run(datasets=datasets, training_params=training_params, base_model_manager=base_model_manager, samples_size=samples_size, num_calibration_runs=num_calibration_runs, ensemble_size=ensemble_size, patience=patience, allow_margin=allow_margin, margin=margin) detectron_results.analyze_results(test_strategies) print("Running Profiled Detectron Experiment:") detectron_profiles_res = MDRCalculator.detectron_by_profiles(datasets=datasets, profiles_manager=testing_3pa_res.get_profiles_manager(),training_params=training_params, base_model_manager=base_model_manager, samples_size=samples_size_profiles, num_calibration_runs=num_calibration_runs, ensemble_size=ensemble_size, patience=patience, strategies=test_strategies, allow_margin=allow_margin, margin=margin, all_dr=all_dr) return reference_3pa_res, testing_3pa_res, detectron_results