cpdbench.utils.UserConfig

  1from cpdbench.task.Task import TaskType
  2
  3
  4def _get_path_to_execution_array(task_type: TaskType) -> str:
  5    """Returns the path in the user config dict to the needed execution variables depending on the task type
  6    :param task_type: the task type for which the path is needed
  7    :return: the path in the user config
  8    """
  9    if task_type == TaskType.DATASET_FETCH:
 10        return "dataset-executions"
 11    elif task_type == TaskType.ALGORITHM_EXECUTION:
 12        return "algorithm-executions"
 13    else:
 14        return "metric-executions"
 15
 16
 17class UserConfig:
 18    """This class represents the part of the global config file which contains all user variables.
 19    There are two types of user parameters/variables in this testbench:
 20    - global variables: variables which can be used in any task, and can only be defined once
 21    - execution variables: specific variables for each task type, for which multiple instances/different values given
 22        as list can exist. In this case, any task using these are executed multiple times depending on the amount of
 23        given instances. A use case for this is executing the same algorithm but with different parameters.
 24    """
 25
 26    def __init__(self, user_params=None):
 27        """Constructor for the UserConfig class
 28        :param user_params: the user params as dict extracted from the config file
 29        """
 30        if user_params is None:
 31            user_params = {}
 32        self._user_param_dict = user_params
 33
 34    def get_number_of_executions(self, tasks_type: TaskType) -> int:
 35        """Returns the needed number of executions of each task for the given type depending on the amount of
 36        parameter sets
 37        :param tasks_type: the task type for which the number should be returned
 38        :return: the number of executions
 39        """
 40        if self._user_param_dict == {}:
 41            return 1
 42        execution_yaml = self._user_param_dict.get(_get_path_to_execution_array(tasks_type))
 43        if execution_yaml is None:
 44            return 1
 45        return len(execution_yaml)
 46
 47    def get_user_param(self, param_name: str, task_type: TaskType) -> list:
 48        """Returns the values of the given user parameter name.
 49        Returns a list because for execution variables multiple instances (for multiple runs) can be defined.
 50        Throws an exception if the parameter is not defined.
 51        :param param_name: the name of the user parameter
 52        :param task_type: the task type if the parameter is an execution variable.
 53        If it is a global variable, this parameter is not used.
 54        :return: a list of all possible values for the requested parameter
 55        """
 56        global_param = self._user_param_dict.get(param_name)
 57        if global_param is not None:
 58            return [global_param]
 59        else:
 60            dict_string = _get_path_to_execution_array(task_type)
 61            try:
 62                exec_list = self._user_param_dict[dict_string]
 63            except KeyError:
 64                raise Exception("Parameter not found")
 65            if exec_list is None:
 66                raise Exception("Parameter not found")
 67            try:
 68                result = [execution_dict[param_name] for execution_dict in exec_list]
 69            except KeyError:
 70                raise Exception("Parameter not found")
 71            else:
 72                if result is None or len(result) == 0:
 73                    raise Exception("Parameter not found")
 74                return result
 75
 76    def check_if_global_param(self, param_name: str) -> bool:
 77        """Checks if the given parameter is a global one or an execution parameter.
 78        :param param_name: the name of the to be checked parameter
 79        :return: True if the parameter is a global one. False otherwise.
 80        """
 81        is_global_param = None
 82        global_param = self._user_param_dict.get(param_name)
 83        if global_param is not None:
 84            is_global_param = True
 85        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
 86            part_dict = self._user_param_dict.get(i)
 87            if part_dict is None or part_dict[0] is None:
 88                continue
 89            user_param = part_dict[0].get(param_name)
 90            if user_param is not None:
 91                if is_global_param is True:
 92                    raise Exception("Parameter both global and execution")
 93                else:
 94                    return False
 95        if is_global_param is None:
 96            raise Exception("Parameter not found: " + param_name)
 97        else:
 98            return is_global_param
 99
100    def get_param_dict(self) -> dict:
101        """Returns all user params as dict for logging purposes.
102        :return: the user param dict"""
103        return self._user_param_dict
104
105    def validate_user_config(self) -> None:
106        """Validates the user config for common errors.
107        This method returns nothing if the validation is successful, and throws errors
108        if something is wrong with the config, as the bench cannot continue with consistency
109        errors in the config.
110        """
111        # execution params are declared correctly
112        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
113            exec_dict = self._user_param_dict.get(i)
114            if exec_dict is None:
115                continue
116            if type(exec_dict) is not list:
117                raise Exception("execution params not declared correctly")
118            for j in exec_dict:
119                if type(j) is not dict:
120                    raise Exception("execution params not declared correctly")
121
122        # execution param is given for all configurations
123        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
124            part_dict = self._user_param_dict.get(i)
125            if part_dict is None:
126                continue
127            exec_params = [param for param in part_dict[0]]
128            for j in range(1, len(part_dict)):
129                for par in exec_params:
130                    if part_dict[j].get(par) is None:
131                        raise Exception("Parameter not found in all configurations: " + par)
132
133        params = self._get_all_params()
134        for param in params:
135            # parameters are defined as both global and execution
136            is_global_param = None
137            global_param = self._user_param_dict.get(param)
138            if global_param is not None:
139                is_global_param = True
140            for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
141                part_dict = self._user_param_dict.get(i)
142                if part_dict is None:
143                    continue
144                user_param = part_dict[0].get(param)
145                if user_param is not None:
146                    if is_global_param is True:
147                        raise Exception(f"Parameter {param} both global and execution")
148
149    def _get_all_params(self) -> set:
150        params = [param for param in self._user_param_dict
151                  if param not in ["dataset-executions", "algorithm-executions", "metric-executions"]]
152        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
153            exec_dict = self._user_param_dict.get(i)
154            if exec_dict is None:
155                continue
156            user_params = [param for param in exec_dict[0]]
157            params = params + user_params
158        return set(params)
class UserConfig:
 18class UserConfig:
 19    """This class represents the part of the global config file which contains all user variables.
 20    There are two types of user parameters/variables in this testbench:
 21    - global variables: variables which can be used in any task, and can only be defined once
 22    - execution variables: specific variables for each task type, for which multiple instances/different values given
 23        as list can exist. In this case, any task using these are executed multiple times depending on the amount of
 24        given instances. A use case for this is executing the same algorithm but with different parameters.
 25    """
 26
 27    def __init__(self, user_params=None):
 28        """Constructor for the UserConfig class
 29        :param user_params: the user params as dict extracted from the config file
 30        """
 31        if user_params is None:
 32            user_params = {}
 33        self._user_param_dict = user_params
 34
 35    def get_number_of_executions(self, tasks_type: TaskType) -> int:
 36        """Returns the needed number of executions of each task for the given type depending on the amount of
 37        parameter sets
 38        :param tasks_type: the task type for which the number should be returned
 39        :return: the number of executions
 40        """
 41        if self._user_param_dict == {}:
 42            return 1
 43        execution_yaml = self._user_param_dict.get(_get_path_to_execution_array(tasks_type))
 44        if execution_yaml is None:
 45            return 1
 46        return len(execution_yaml)
 47
 48    def get_user_param(self, param_name: str, task_type: TaskType) -> list:
 49        """Returns the values of the given user parameter name.
 50        Returns a list because for execution variables multiple instances (for multiple runs) can be defined.
 51        Throws an exception if the parameter is not defined.
 52        :param param_name: the name of the user parameter
 53        :param task_type: the task type if the parameter is an execution variable.
 54        If it is a global variable, this parameter is not used.
 55        :return: a list of all possible values for the requested parameter
 56        """
 57        global_param = self._user_param_dict.get(param_name)
 58        if global_param is not None:
 59            return [global_param]
 60        else:
 61            dict_string = _get_path_to_execution_array(task_type)
 62            try:
 63                exec_list = self._user_param_dict[dict_string]
 64            except KeyError:
 65                raise Exception("Parameter not found")
 66            if exec_list is None:
 67                raise Exception("Parameter not found")
 68            try:
 69                result = [execution_dict[param_name] for execution_dict in exec_list]
 70            except KeyError:
 71                raise Exception("Parameter not found")
 72            else:
 73                if result is None or len(result) == 0:
 74                    raise Exception("Parameter not found")
 75                return result
 76
 77    def check_if_global_param(self, param_name: str) -> bool:
 78        """Checks if the given parameter is a global one or an execution parameter.
 79        :param param_name: the name of the to be checked parameter
 80        :return: True if the parameter is a global one. False otherwise.
 81        """
 82        is_global_param = None
 83        global_param = self._user_param_dict.get(param_name)
 84        if global_param is not None:
 85            is_global_param = True
 86        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
 87            part_dict = self._user_param_dict.get(i)
 88            if part_dict is None or part_dict[0] is None:
 89                continue
 90            user_param = part_dict[0].get(param_name)
 91            if user_param is not None:
 92                if is_global_param is True:
 93                    raise Exception("Parameter both global and execution")
 94                else:
 95                    return False
 96        if is_global_param is None:
 97            raise Exception("Parameter not found: " + param_name)
 98        else:
 99            return is_global_param
100
101    def get_param_dict(self) -> dict:
102        """Returns all user params as dict for logging purposes.
103        :return: the user param dict"""
104        return self._user_param_dict
105
106    def validate_user_config(self) -> None:
107        """Validates the user config for common errors.
108        This method returns nothing if the validation is successful, and throws errors
109        if something is wrong with the config, as the bench cannot continue with consistency
110        errors in the config.
111        """
112        # execution params are declared correctly
113        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
114            exec_dict = self._user_param_dict.get(i)
115            if exec_dict is None:
116                continue
117            if type(exec_dict) is not list:
118                raise Exception("execution params not declared correctly")
119            for j in exec_dict:
120                if type(j) is not dict:
121                    raise Exception("execution params not declared correctly")
122
123        # execution param is given for all configurations
124        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
125            part_dict = self._user_param_dict.get(i)
126            if part_dict is None:
127                continue
128            exec_params = [param for param in part_dict[0]]
129            for j in range(1, len(part_dict)):
130                for par in exec_params:
131                    if part_dict[j].get(par) is None:
132                        raise Exception("Parameter not found in all configurations: " + par)
133
134        params = self._get_all_params()
135        for param in params:
136            # parameters are defined as both global and execution
137            is_global_param = None
138            global_param = self._user_param_dict.get(param)
139            if global_param is not None:
140                is_global_param = True
141            for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
142                part_dict = self._user_param_dict.get(i)
143                if part_dict is None:
144                    continue
145                user_param = part_dict[0].get(param)
146                if user_param is not None:
147                    if is_global_param is True:
148                        raise Exception(f"Parameter {param} both global and execution")
149
150    def _get_all_params(self) -> set:
151        params = [param for param in self._user_param_dict
152                  if param not in ["dataset-executions", "algorithm-executions", "metric-executions"]]
153        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
154            exec_dict = self._user_param_dict.get(i)
155            if exec_dict is None:
156                continue
157            user_params = [param for param in exec_dict[0]]
158            params = params + user_params
159        return set(params)

This class represents the part of the global config file which contains all user variables. There are two types of user parameters/variables in this testbench:

  • global variables: variables which can be used in any task, and can only be defined once
  • execution variables: specific variables for each task type, for which multiple instances/different values given as list can exist. In this case, any task using these are executed multiple times depending on the amount of given instances. A use case for this is executing the same algorithm but with different parameters.
UserConfig(user_params=None)
27    def __init__(self, user_params=None):
28        """Constructor for the UserConfig class
29        :param user_params: the user params as dict extracted from the config file
30        """
31        if user_params is None:
32            user_params = {}
33        self._user_param_dict = user_params

Constructor for the UserConfig class

Parameters
  • user_params: the user params as dict extracted from the config file
def get_number_of_executions(self, tasks_type: cpdbench.task.Task.TaskType) -> int:
35    def get_number_of_executions(self, tasks_type: TaskType) -> int:
36        """Returns the needed number of executions of each task for the given type depending on the amount of
37        parameter sets
38        :param tasks_type: the task type for which the number should be returned
39        :return: the number of executions
40        """
41        if self._user_param_dict == {}:
42            return 1
43        execution_yaml = self._user_param_dict.get(_get_path_to_execution_array(tasks_type))
44        if execution_yaml is None:
45            return 1
46        return len(execution_yaml)

Returns the needed number of executions of each task for the given type depending on the amount of parameter sets

Parameters
  • tasks_type: the task type for which the number should be returned
Returns

the number of executions

def get_user_param(self, param_name: str, task_type: cpdbench.task.Task.TaskType) -> list:
48    def get_user_param(self, param_name: str, task_type: TaskType) -> list:
49        """Returns the values of the given user parameter name.
50        Returns a list because for execution variables multiple instances (for multiple runs) can be defined.
51        Throws an exception if the parameter is not defined.
52        :param param_name: the name of the user parameter
53        :param task_type: the task type if the parameter is an execution variable.
54        If it is a global variable, this parameter is not used.
55        :return: a list of all possible values for the requested parameter
56        """
57        global_param = self._user_param_dict.get(param_name)
58        if global_param is not None:
59            return [global_param]
60        else:
61            dict_string = _get_path_to_execution_array(task_type)
62            try:
63                exec_list = self._user_param_dict[dict_string]
64            except KeyError:
65                raise Exception("Parameter not found")
66            if exec_list is None:
67                raise Exception("Parameter not found")
68            try:
69                result = [execution_dict[param_name] for execution_dict in exec_list]
70            except KeyError:
71                raise Exception("Parameter not found")
72            else:
73                if result is None or len(result) == 0:
74                    raise Exception("Parameter not found")
75                return result

Returns the values of the given user parameter name. Returns a list because for execution variables multiple instances (for multiple runs) can be defined. Throws an exception if the parameter is not defined.

Parameters
  • param_name: the name of the user parameter
  • task_type: the task type if the parameter is an execution variable. If it is a global variable, this parameter is not used.
Returns

a list of all possible values for the requested parameter

def check_if_global_param(self, param_name: str) -> bool:
77    def check_if_global_param(self, param_name: str) -> bool:
78        """Checks if the given parameter is a global one or an execution parameter.
79        :param param_name: the name of the to be checked parameter
80        :return: True if the parameter is a global one. False otherwise.
81        """
82        is_global_param = None
83        global_param = self._user_param_dict.get(param_name)
84        if global_param is not None:
85            is_global_param = True
86        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
87            part_dict = self._user_param_dict.get(i)
88            if part_dict is None or part_dict[0] is None:
89                continue
90            user_param = part_dict[0].get(param_name)
91            if user_param is not None:
92                if is_global_param is True:
93                    raise Exception("Parameter both global and execution")
94                else:
95                    return False
96        if is_global_param is None:
97            raise Exception("Parameter not found: " + param_name)
98        else:
99            return is_global_param

Checks if the given parameter is a global one or an execution parameter.

Parameters
  • param_name: the name of the to be checked parameter
Returns

True if the parameter is a global one. False otherwise.

def get_param_dict(self) -> dict:
101    def get_param_dict(self) -> dict:
102        """Returns all user params as dict for logging purposes.
103        :return: the user param dict"""
104        return self._user_param_dict

Returns all user params as dict for logging purposes.

Returns

the user param dict

def validate_user_config(self) -> None:
106    def validate_user_config(self) -> None:
107        """Validates the user config for common errors.
108        This method returns nothing if the validation is successful, and throws errors
109        if something is wrong with the config, as the bench cannot continue with consistency
110        errors in the config.
111        """
112        # execution params are declared correctly
113        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
114            exec_dict = self._user_param_dict.get(i)
115            if exec_dict is None:
116                continue
117            if type(exec_dict) is not list:
118                raise Exception("execution params not declared correctly")
119            for j in exec_dict:
120                if type(j) is not dict:
121                    raise Exception("execution params not declared correctly")
122
123        # execution param is given for all configurations
124        for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
125            part_dict = self._user_param_dict.get(i)
126            if part_dict is None:
127                continue
128            exec_params = [param for param in part_dict[0]]
129            for j in range(1, len(part_dict)):
130                for par in exec_params:
131                    if part_dict[j].get(par) is None:
132                        raise Exception("Parameter not found in all configurations: " + par)
133
134        params = self._get_all_params()
135        for param in params:
136            # parameters are defined as both global and execution
137            is_global_param = None
138            global_param = self._user_param_dict.get(param)
139            if global_param is not None:
140                is_global_param = True
141            for i in ["dataset-executions", "algorithm-executions", "metric-executions"]:
142                part_dict = self._user_param_dict.get(i)
143                if part_dict is None:
144                    continue
145                user_param = part_dict[0].get(param)
146                if user_param is not None:
147                    if is_global_param is True:
148                        raise Exception(f"Parameter {param} both global and execution")

Validates the user config for common errors. This method returns nothing if the validation is successful, and throws errors if something is wrong with the config, as the bench cannot continue with consistency errors in the config.