cpdbench.utils.UserConfig
1from cpdbench.task.Task import TaskType 2 3 4def _get_path_to_execution_array(task_type: TaskType) -> str: 5 """Returns the path in the user config dict to the needed execution variables depending on the task type 6 :param task_type: the task type for which the path is needed 7 :return: the path in the user config 8 """ 9 if task_type == TaskType.DATASET_FETCH: 10 return "dataset-executions" 11 elif task_type == TaskType.ALGORITHM_EXECUTION: 12 return "algorithm-executions" 13 else: 14 return "metric-executions" 15 16 17class UserConfig: 18 """This class represents the part of the global config file which contains all user variables. 19 There are two types of user parameters/variables in this testbench: 20 - global variables: variables which can be used in any task, and can only be defined once 21 - execution variables: specific variables for each task type, for which multiple instances/different values given 22 as list can exist. In this case, any task using these are executed multiple times depending on the amount of 23 given instances. A use case for this is executing the same algorithm but with different parameters. 24 """ 25 26 def __init__(self, user_params=None): 27 """Constructor for the UserConfig class 28 :param user_params: the user params as dict extracted from the config file 29 """ 30 if user_params is None: 31 user_params = {} 32 self._user_param_dict = user_params 33 34 def get_number_of_executions(self, tasks_type: TaskType) -> int: 35 """Returns the needed number of executions of each task for the given type depending on the amount of 36 parameter sets 37 :param tasks_type: the task type for which the number should be returned 38 :return: the number of executions 39 """ 40 if self._user_param_dict == {}: 41 return 1 42 execution_yaml = self._user_param_dict.get(_get_path_to_execution_array(tasks_type)) 43 if execution_yaml is None: 44 return 1 45 return len(execution_yaml) 46 47 def get_user_param(self, param_name: str, task_type: TaskType) -> list: 48 """Returns the values of the given user parameter name. 49 Returns a list because for execution variables multiple instances (for multiple runs) can be defined. 50 Throws an exception if the parameter is not defined. 51 :param param_name: the name of the user parameter 52 :param task_type: the task type if the parameter is an execution variable. 53 If it is a global variable, this parameter is not used. 54 :return: a list of all possible values for the requested parameter 55 """ 56 global_param = self._user_param_dict.get(param_name) 57 if global_param is not None: 58 return [global_param] 59 else: 60 dict_string = _get_path_to_execution_array(task_type) 61 try: 62 exec_list = self._user_param_dict[dict_string] 63 except KeyError: 64 raise Exception("Parameter not found") 65 if exec_list is None: 66 raise Exception("Parameter not found") 67 try: 68 result = [execution_dict[param_name] for execution_dict in exec_list] 69 except KeyError: 70 raise Exception("Parameter not found") 71 else: 72 if result is None or len(result) == 0: 73 raise Exception("Parameter not found") 74 return result 75 76 def check_if_global_param(self, param_name: str) -> bool: 77 """Checks if the given parameter is a global one or an execution parameter. 78 :param param_name: the name of the to be checked parameter 79 :return: True if the parameter is a global one. False otherwise. 80 """ 81 is_global_param = None 82 global_param = self._user_param_dict.get(param_name) 83 if global_param is not None: 84 is_global_param = True 85 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 86 part_dict = self._user_param_dict.get(i) 87 if part_dict is None or part_dict[0] is None: 88 continue 89 user_param = part_dict[0].get(param_name) 90 if user_param is not None: 91 if is_global_param is True: 92 raise Exception("Parameter both global and execution") 93 else: 94 return False 95 if is_global_param is None: 96 raise Exception("Parameter not found: " + param_name) 97 else: 98 return is_global_param 99 100 def get_param_dict(self) -> dict: 101 """Returns all user params as dict for logging purposes. 102 :return: the user param dict""" 103 return self._user_param_dict 104 105 def validate_user_config(self) -> None: 106 """Validates the user config for common errors. 107 This method returns nothing if the validation is successful, and throws errors 108 if something is wrong with the config, as the bench cannot continue with consistency 109 errors in the config. 110 """ 111 # execution params are declared correctly 112 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 113 exec_dict = self._user_param_dict.get(i) 114 if exec_dict is None: 115 continue 116 if type(exec_dict) is not list: 117 raise Exception("execution params not declared correctly") 118 for j in exec_dict: 119 if type(j) is not dict: 120 raise Exception("execution params not declared correctly") 121 122 # execution param is given for all configurations 123 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 124 part_dict = self._user_param_dict.get(i) 125 if part_dict is None: 126 continue 127 exec_params = [param for param in part_dict[0]] 128 for j in range(1, len(part_dict)): 129 for par in exec_params: 130 if part_dict[j].get(par) is None: 131 raise Exception("Parameter not found in all configurations: " + par) 132 133 params = self._get_all_params() 134 for param in params: 135 # parameters are defined as both global and execution 136 is_global_param = None 137 global_param = self._user_param_dict.get(param) 138 if global_param is not None: 139 is_global_param = True 140 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 141 part_dict = self._user_param_dict.get(i) 142 if part_dict is None: 143 continue 144 user_param = part_dict[0].get(param) 145 if user_param is not None: 146 if is_global_param is True: 147 raise Exception(f"Parameter {param} both global and execution") 148 149 def _get_all_params(self) -> set: 150 params = [param for param in self._user_param_dict 151 if param not in ["dataset-executions", "algorithm-executions", "metric-executions"]] 152 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 153 exec_dict = self._user_param_dict.get(i) 154 if exec_dict is None: 155 continue 156 user_params = [param for param in exec_dict[0]] 157 params = params + user_params 158 return set(params)
18class UserConfig: 19 """This class represents the part of the global config file which contains all user variables. 20 There are two types of user parameters/variables in this testbench: 21 - global variables: variables which can be used in any task, and can only be defined once 22 - execution variables: specific variables for each task type, for which multiple instances/different values given 23 as list can exist. In this case, any task using these are executed multiple times depending on the amount of 24 given instances. A use case for this is executing the same algorithm but with different parameters. 25 """ 26 27 def __init__(self, user_params=None): 28 """Constructor for the UserConfig class 29 :param user_params: the user params as dict extracted from the config file 30 """ 31 if user_params is None: 32 user_params = {} 33 self._user_param_dict = user_params 34 35 def get_number_of_executions(self, tasks_type: TaskType) -> int: 36 """Returns the needed number of executions of each task for the given type depending on the amount of 37 parameter sets 38 :param tasks_type: the task type for which the number should be returned 39 :return: the number of executions 40 """ 41 if self._user_param_dict == {}: 42 return 1 43 execution_yaml = self._user_param_dict.get(_get_path_to_execution_array(tasks_type)) 44 if execution_yaml is None: 45 return 1 46 return len(execution_yaml) 47 48 def get_user_param(self, param_name: str, task_type: TaskType) -> list: 49 """Returns the values of the given user parameter name. 50 Returns a list because for execution variables multiple instances (for multiple runs) can be defined. 51 Throws an exception if the parameter is not defined. 52 :param param_name: the name of the user parameter 53 :param task_type: the task type if the parameter is an execution variable. 54 If it is a global variable, this parameter is not used. 55 :return: a list of all possible values for the requested parameter 56 """ 57 global_param = self._user_param_dict.get(param_name) 58 if global_param is not None: 59 return [global_param] 60 else: 61 dict_string = _get_path_to_execution_array(task_type) 62 try: 63 exec_list = self._user_param_dict[dict_string] 64 except KeyError: 65 raise Exception("Parameter not found") 66 if exec_list is None: 67 raise Exception("Parameter not found") 68 try: 69 result = [execution_dict[param_name] for execution_dict in exec_list] 70 except KeyError: 71 raise Exception("Parameter not found") 72 else: 73 if result is None or len(result) == 0: 74 raise Exception("Parameter not found") 75 return result 76 77 def check_if_global_param(self, param_name: str) -> bool: 78 """Checks if the given parameter is a global one or an execution parameter. 79 :param param_name: the name of the to be checked parameter 80 :return: True if the parameter is a global one. False otherwise. 81 """ 82 is_global_param = None 83 global_param = self._user_param_dict.get(param_name) 84 if global_param is not None: 85 is_global_param = True 86 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 87 part_dict = self._user_param_dict.get(i) 88 if part_dict is None or part_dict[0] is None: 89 continue 90 user_param = part_dict[0].get(param_name) 91 if user_param is not None: 92 if is_global_param is True: 93 raise Exception("Parameter both global and execution") 94 else: 95 return False 96 if is_global_param is None: 97 raise Exception("Parameter not found: " + param_name) 98 else: 99 return is_global_param 100 101 def get_param_dict(self) -> dict: 102 """Returns all user params as dict for logging purposes. 103 :return: the user param dict""" 104 return self._user_param_dict 105 106 def validate_user_config(self) -> None: 107 """Validates the user config for common errors. 108 This method returns nothing if the validation is successful, and throws errors 109 if something is wrong with the config, as the bench cannot continue with consistency 110 errors in the config. 111 """ 112 # execution params are declared correctly 113 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 114 exec_dict = self._user_param_dict.get(i) 115 if exec_dict is None: 116 continue 117 if type(exec_dict) is not list: 118 raise Exception("execution params not declared correctly") 119 for j in exec_dict: 120 if type(j) is not dict: 121 raise Exception("execution params not declared correctly") 122 123 # execution param is given for all configurations 124 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 125 part_dict = self._user_param_dict.get(i) 126 if part_dict is None: 127 continue 128 exec_params = [param for param in part_dict[0]] 129 for j in range(1, len(part_dict)): 130 for par in exec_params: 131 if part_dict[j].get(par) is None: 132 raise Exception("Parameter not found in all configurations: " + par) 133 134 params = self._get_all_params() 135 for param in params: 136 # parameters are defined as both global and execution 137 is_global_param = None 138 global_param = self._user_param_dict.get(param) 139 if global_param is not None: 140 is_global_param = True 141 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 142 part_dict = self._user_param_dict.get(i) 143 if part_dict is None: 144 continue 145 user_param = part_dict[0].get(param) 146 if user_param is not None: 147 if is_global_param is True: 148 raise Exception(f"Parameter {param} both global and execution") 149 150 def _get_all_params(self) -> set: 151 params = [param for param in self._user_param_dict 152 if param not in ["dataset-executions", "algorithm-executions", "metric-executions"]] 153 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 154 exec_dict = self._user_param_dict.get(i) 155 if exec_dict is None: 156 continue 157 user_params = [param for param in exec_dict[0]] 158 params = params + user_params 159 return set(params)
This class represents the part of the global config file which contains all user variables. There are two types of user parameters/variables in this testbench:
- global variables: variables which can be used in any task, and can only be defined once
- execution variables: specific variables for each task type, for which multiple instances/different values given as list can exist. In this case, any task using these are executed multiple times depending on the amount of given instances. A use case for this is executing the same algorithm but with different parameters.
27 def __init__(self, user_params=None): 28 """Constructor for the UserConfig class 29 :param user_params: the user params as dict extracted from the config file 30 """ 31 if user_params is None: 32 user_params = {} 33 self._user_param_dict = user_params
Constructor for the UserConfig class
Parameters
- user_params: the user params as dict extracted from the config file
35 def get_number_of_executions(self, tasks_type: TaskType) -> int: 36 """Returns the needed number of executions of each task for the given type depending on the amount of 37 parameter sets 38 :param tasks_type: the task type for which the number should be returned 39 :return: the number of executions 40 """ 41 if self._user_param_dict == {}: 42 return 1 43 execution_yaml = self._user_param_dict.get(_get_path_to_execution_array(tasks_type)) 44 if execution_yaml is None: 45 return 1 46 return len(execution_yaml)
Returns the needed number of executions of each task for the given type depending on the amount of parameter sets
Parameters
- tasks_type: the task type for which the number should be returned
Returns
the number of executions
48 def get_user_param(self, param_name: str, task_type: TaskType) -> list: 49 """Returns the values of the given user parameter name. 50 Returns a list because for execution variables multiple instances (for multiple runs) can be defined. 51 Throws an exception if the parameter is not defined. 52 :param param_name: the name of the user parameter 53 :param task_type: the task type if the parameter is an execution variable. 54 If it is a global variable, this parameter is not used. 55 :return: a list of all possible values for the requested parameter 56 """ 57 global_param = self._user_param_dict.get(param_name) 58 if global_param is not None: 59 return [global_param] 60 else: 61 dict_string = _get_path_to_execution_array(task_type) 62 try: 63 exec_list = self._user_param_dict[dict_string] 64 except KeyError: 65 raise Exception("Parameter not found") 66 if exec_list is None: 67 raise Exception("Parameter not found") 68 try: 69 result = [execution_dict[param_name] for execution_dict in exec_list] 70 except KeyError: 71 raise Exception("Parameter not found") 72 else: 73 if result is None or len(result) == 0: 74 raise Exception("Parameter not found") 75 return result
Returns the values of the given user parameter name. Returns a list because for execution variables multiple instances (for multiple runs) can be defined. Throws an exception if the parameter is not defined.
Parameters
- param_name: the name of the user parameter
- task_type: the task type if the parameter is an execution variable. If it is a global variable, this parameter is not used.
Returns
a list of all possible values for the requested parameter
77 def check_if_global_param(self, param_name: str) -> bool: 78 """Checks if the given parameter is a global one or an execution parameter. 79 :param param_name: the name of the to be checked parameter 80 :return: True if the parameter is a global one. False otherwise. 81 """ 82 is_global_param = None 83 global_param = self._user_param_dict.get(param_name) 84 if global_param is not None: 85 is_global_param = True 86 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 87 part_dict = self._user_param_dict.get(i) 88 if part_dict is None or part_dict[0] is None: 89 continue 90 user_param = part_dict[0].get(param_name) 91 if user_param is not None: 92 if is_global_param is True: 93 raise Exception("Parameter both global and execution") 94 else: 95 return False 96 if is_global_param is None: 97 raise Exception("Parameter not found: " + param_name) 98 else: 99 return is_global_param
Checks if the given parameter is a global one or an execution parameter.
Parameters
- param_name: the name of the to be checked parameter
Returns
True if the parameter is a global one. False otherwise.
101 def get_param_dict(self) -> dict: 102 """Returns all user params as dict for logging purposes. 103 :return: the user param dict""" 104 return self._user_param_dict
Returns all user params as dict for logging purposes.
Returns
the user param dict
106 def validate_user_config(self) -> None: 107 """Validates the user config for common errors. 108 This method returns nothing if the validation is successful, and throws errors 109 if something is wrong with the config, as the bench cannot continue with consistency 110 errors in the config. 111 """ 112 # execution params are declared correctly 113 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 114 exec_dict = self._user_param_dict.get(i) 115 if exec_dict is None: 116 continue 117 if type(exec_dict) is not list: 118 raise Exception("execution params not declared correctly") 119 for j in exec_dict: 120 if type(j) is not dict: 121 raise Exception("execution params not declared correctly") 122 123 # execution param is given for all configurations 124 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 125 part_dict = self._user_param_dict.get(i) 126 if part_dict is None: 127 continue 128 exec_params = [param for param in part_dict[0]] 129 for j in range(1, len(part_dict)): 130 for par in exec_params: 131 if part_dict[j].get(par) is None: 132 raise Exception("Parameter not found in all configurations: " + par) 133 134 params = self._get_all_params() 135 for param in params: 136 # parameters are defined as both global and execution 137 is_global_param = None 138 global_param = self._user_param_dict.get(param) 139 if global_param is not None: 140 is_global_param = True 141 for i in ["dataset-executions", "algorithm-executions", "metric-executions"]: 142 part_dict = self._user_param_dict.get(i) 143 if part_dict is None: 144 continue 145 user_param = part_dict[0].get(param) 146 if user_param is not None: 147 if is_global_param is True: 148 raise Exception(f"Parameter {param} both global and execution")
Validates the user config for common errors. This method returns nothing if the validation is successful, and throws errors if something is wrong with the config, as the bench cannot continue with consistency errors in the config.