Coverage for src/hdmf/build/classgenerator.py: 97%
199 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-07-25 05:02 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-07-25 05:02 +0000
1from copy import deepcopy
2from datetime import datetime, date
4import numpy as np
6from ..container import Container, Data, DataRegion, MultiContainerInterface
7from ..spec import AttributeSpec, LinkSpec, RefSpec, GroupSpec
8from ..spec.spec import BaseStorageSpec, ZERO_OR_MANY, ONE_OR_MANY
9from ..utils import docval, getargs, ExtenderMeta, get_docval, popargs, AllowPositional
12class ClassGenerator:
14 def __init__(self):
15 self.__custom_generators = []
17 @property
18 def custom_generators(self):
19 return self.__custom_generators
21 @docval({'name': 'generator', 'type': type, 'doc': 'the CustomClassGenerator class to register'})
22 def register_generator(self, **kwargs):
23 """Add a custom class generator to this ClassGenerator.
25 Generators added later are run first. Duplicates are moved to the top of the list.
26 """
27 generator = getargs('generator', kwargs)
28 if not issubclass(generator, CustomClassGenerator):
29 raise ValueError('Generator %s must be a subclass of CustomClassGenerator.' % generator)
30 if generator in self.__custom_generators:
31 self.__custom_generators.remove(generator)
32 self.__custom_generators.insert(0, generator)
34 @docval({'name': 'data_type', 'type': str, 'doc': 'the data type to create a AbstractContainer class for'},
35 {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''},
36 {'name': 'parent_cls', 'type': type, 'doc': ''},
37 {'name': 'attr_names', 'type': dict, 'doc': ''},
38 {'name': 'type_map', 'type': 'TypeMap', 'doc': ''},
39 returns='the class for the given namespace and data_type', rtype=type)
40 def generate_class(self, **kwargs):
41 """Get the container class from data type specification.
42 If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically
43 created and returned.
44 """
45 data_type, spec, parent_cls, attr_names, type_map = getargs('data_type', 'spec', 'parent_cls', 'attr_names',
46 'type_map', kwargs)
48 not_inherited_fields = dict()
49 for k, field_spec in attr_names.items():
50 if k == 'help': # pragma: no cover
51 # (legacy) do not add field named 'help' to any part of class object
52 continue
53 if isinstance(field_spec, GroupSpec) and field_spec.data_type is None: # skip named, untyped groups 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true
54 continue
55 if not spec.is_inherited_spec(field_spec):
56 not_inherited_fields[k] = field_spec
57 try:
58 classdict = dict()
59 bases = [parent_cls]
60 docval_args = list(deepcopy(get_docval(parent_cls.__init__)))
61 for attr_name, field_spec in not_inherited_fields.items():
62 for class_generator in self.__custom_generators: # pragma: no branch
63 # each generator can update classdict and docval_args
64 if class_generator.apply_generator_to_field(field_spec, bases, type_map):
65 class_generator.process_field_spec(classdict, docval_args, parent_cls, attr_name,
66 not_inherited_fields, type_map, spec)
67 break # each field_spec should be processed by only one generator
69 for class_generator in self.__custom_generators:
70 class_generator.post_process(classdict, bases, docval_args, spec)
72 for class_generator in reversed(self.__custom_generators):
73 # go in reverse order so that base init is added first and
74 # later class generators can modify or overwrite __init__ set by an earlier class generator
75 class_generator.set_init(classdict, bases, docval_args, not_inherited_fields, spec.name)
76 except TypeDoesNotExistError as e: # pragma: no cover
77 # this error should never happen after hdmf#322
78 name = spec.data_type_def
79 if name is None:
80 name = 'Unknown'
81 raise ValueError("Cannot dynamically generate class for type '%s'. " % name
82 + str(e)
83 + " Please define that type before defining '%s'." % name)
84 cls = ExtenderMeta(data_type, tuple(bases), classdict)
85 return cls
88class TypeDoesNotExistError(Exception): # pragma: no cover
89 pass
92class CustomClassGenerator:
93 """Subclass this class and register an instance to alter how classes are auto-generated."""
95 def __new__(cls, *args, **kwargs): # pragma: no cover
96 raise TypeError('Cannot instantiate class %s' % cls.__name__)
98 # mapping from spec types to allowable python types for docval for fields during dynamic class generation
99 # e.g., if a dataset/attribute spec has dtype int32, then get_class should generate a docval for the class'
100 # __init__ method that allows the types (int, np.int32, np.int64) for the corresponding field.
101 # passing an np.int16 would raise a docval error.
102 # passing an int64 to __init__ would result in the field storing the value as an int64 (and subsequently written
103 # as an int64). no upconversion or downconversion happens as a result of this map
104 _spec_dtype_map = {
105 'float32': (float, np.float32, np.float64),
106 'float': (float, np.float32, np.float64),
107 'float64': (float, np.float64),
108 'double': (float, np.float64),
109 'int8': (np.int8, np.int16, np.int32, np.int64, int),
110 'int16': (np.int16, np.int32, np.int64, int),
111 'short': (np.int16, np.int32, np.int64, int),
112 'int32': (int, np.int32, np.int64),
113 'int': (int, np.int32, np.int64),
114 'int64': np.int64,
115 'long': np.int64,
116 'uint8': (np.uint8, np.uint16, np.uint32, np.uint64),
117 'uint16': (np.uint16, np.uint32, np.uint64),
118 'uint32': (np.uint32, np.uint64),
119 'uint64': np.uint64,
120 'numeric': (float, np.float32, np.float64, np.int8, np.int16, np.int32, np.int64, int, np.uint8, np.uint16,
121 np.uint32, np.uint64),
122 'text': str,
123 'utf': str,
124 'utf8': str,
125 'utf-8': str,
126 'ascii': bytes,
127 'bytes': bytes,
128 'bool': (bool, np.bool_),
129 'isodatetime': (datetime, date),
130 'datetime': (datetime, date)
131 }
133 @classmethod
134 def _get_type_from_spec_dtype(cls, spec_dtype):
135 """Get the Python type associated with the given spec dtype string.
136 Raises ValueError if the given dtype has no mapping to a Python type.
137 """
138 dtype = cls._spec_dtype_map.get(spec_dtype)
139 if dtype is None: # pragma: no cover
140 # this should not happen as long as _spec_dtype_map is kept up to date with
141 # hdmf.spec.spec.DtypeHelper.valid_primary_dtypes
142 raise ValueError("Spec dtype '%s' cannot be mapped to a Python type." % spec_dtype)
143 return dtype
145 @classmethod
146 def _get_container_type(cls, type_name, type_map):
147 """Search all namespaces for the container class associated with the given data type.
148 Raises TypeDoesNotExistError if type is not found in any namespace.
149 """
150 container_type = type_map.get_dt_container_cls(type_name)
151 if container_type is None: # pragma: no cover
152 # this should never happen after hdmf#322
153 raise TypeDoesNotExistError("Type '%s' does not exist." % type_name)
154 return container_type
156 @classmethod
157 def _get_type(cls, spec, type_map):
158 """Get the type of a spec for use in docval.
159 Returns a container class, a type, a tuple of types, ('array_data', 'data') for specs with
160 non-scalar shape, or (Data, Container) when an attribute reference target has not been mapped to a container
161 class.
162 """
163 if isinstance(spec, AttributeSpec):
164 if isinstance(spec.dtype, RefSpec): 164 ↛ 165line 164 didn't jump to line 165, because the condition on line 164 was never true
165 try:
166 container_type = cls._get_container_type(spec.dtype.target_type, type_map)
167 return container_type
168 except TypeDoesNotExistError:
169 # TODO what happens when the attribute ref target is not (or not yet) mapped to a container class?
170 # returning Data, Container works as a generic fallback for now but should be more specific
171 return Data, Container
172 elif spec.shape is None and spec.dims is None:
173 return cls._get_type_from_spec_dtype(spec.dtype)
174 else:
175 return 'array_data', 'data'
176 if isinstance(spec, LinkSpec):
177 return cls._get_container_type(spec.target_type, type_map)
178 if spec.data_type is not None:
179 return cls._get_container_type(spec.data_type, type_map)
180 if spec.shape is None and spec.dims is None:
181 return cls._get_type_from_spec_dtype(spec.dtype)
182 return 'array_data', 'data'
184 @classmethod
185 def _ischild(cls, dtype):
186 """Check if dtype represents a type that is a child."""
187 ret = False
188 if isinstance(dtype, tuple):
189 for sub in dtype:
190 ret = ret or cls._ischild(sub)
191 elif isinstance(dtype, type) and issubclass(dtype, (Container, Data, DataRegion)):
192 ret = True
193 return ret
195 @staticmethod
196 def _set_default_name(docval_args, default_name):
197 """Set the default value for the name docval argument."""
198 if default_name is not None:
199 for x in docval_args:
200 if x['name'] == 'name':
201 x['default'] = default_name
203 @classmethod
204 def apply_generator_to_field(cls, field_spec, bases, type_map):
205 """Return True to signal that this generator should return on all fields not yet processed."""
206 return True
208 @classmethod
209 def process_field_spec(cls, classdict, docval_args, parent_cls, attr_name, not_inherited_fields, type_map, spec):
210 """Add __fields__ to the classdict and update the docval args for the field spec with the given attribute name.
211 :param classdict: The dict to update with __fields__ (or a different parent_cls._fieldsname).
212 :param docval_args: The list of docval arguments.
213 :param parent_cls: The parent class.
214 :param attr_name: The attribute name of the field spec for the container class to generate.
215 :param not_inherited_fields: Dictionary of fields not inherited from the parent class.
216 :param type_map: The type map to use.
217 :param spec: The spec for the container class to generate.
218 """
219 field_spec = not_inherited_fields[attr_name]
220 dtype = cls._get_type(field_spec, type_map)
221 fields_conf = {'name': attr_name,
222 'doc': field_spec['doc']}
223 if cls._ischild(dtype) and issubclass(parent_cls, Container) and not isinstance(field_spec, LinkSpec):
224 fields_conf['child'] = True
225 # if getattr(field_spec, 'value', None) is not None: # TODO set the fixed value on the class?
226 # fields_conf['settable'] = False
227 classdict.setdefault(parent_cls._fieldsname, list()).append(fields_conf)
229 docval_arg = dict(
230 name=attr_name,
231 doc=field_spec.doc,
232 type=cls._get_type(field_spec, type_map)
233 )
234 shape = getattr(field_spec, 'shape', None)
235 if shape is not None:
236 docval_arg['shape'] = shape
237 if cls._check_spec_optional(field_spec, spec):
238 docval_arg['default'] = getattr(field_spec, 'default_value', None)
239 cls._add_to_docval_args(docval_args, docval_arg)
241 @classmethod
242 def _check_spec_optional(cls, field_spec, spec):
243 """Returns True if the spec or any of its parents (up to the parent type spec) are optional."""
244 if not field_spec.required:
245 return True
246 if field_spec == spec:
247 return False
248 if field_spec.parent is not None:
249 return cls._check_spec_optional(field_spec.parent, spec)
251 @classmethod
252 def _add_to_docval_args(cls, docval_args, arg, err_if_present=False):
253 """Add the docval arg to the list if not present. If present, overwrite it in place or raise an error."""
254 inserted = False
255 for i, x in enumerate(docval_args):
256 if x['name'] == arg['name']:
257 if err_if_present: 257 ↛ 258line 257 didn't jump to line 258, because the condition on line 257 was never true
258 raise ValueError("Argument %s already exists in docval args." % arg["name"])
259 docval_args[i] = arg
260 inserted = True
261 if not inserted:
262 docval_args.append(arg)
264 @classmethod
265 def post_process(cls, classdict, bases, docval_args, spec):
266 """Convert classdict['__fields__'] to tuple and update docval args for a fixed name and default name.
267 :param classdict: The class dictionary to convert with '__fields__' key (or a different bases[0]._fieldsname)
268 :param bases: The list of base classes.
269 :param docval_args: The dict of docval arguments.
270 :param spec: The spec for the container class to generate.
271 """
272 # convert classdict['__fields__'] from list to tuple if present
273 for b in bases:
274 fields = classdict.get(b._fieldsname)
275 if fields is not None and not isinstance(fields, tuple):
276 classdict[b._fieldsname] = tuple(fields)
278 # if spec provides a fixed name for this type, remove the 'name' arg from docval_args so that values cannot
279 # be passed for a name positional or keyword arg
280 if spec.name is not None:
281 for arg in list(docval_args):
282 if arg['name'] == 'name':
283 docval_args.remove(arg)
285 # set default name in docval args if provided
286 cls._set_default_name(docval_args, spec.default_name)
288 @classmethod
289 def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name):
290 # get docval arg names from superclass
291 base = bases[0]
292 parent_docval_args = set(arg['name'] for arg in get_docval(base.__init__))
293 new_args = list()
294 for attr_name, field_spec in not_inherited_fields.items():
295 # store arguments for fields that are not in the superclass and not in the superclass __init__ docval
296 # so that they are set after calling base.__init__
297 if attr_name not in parent_docval_args:
298 new_args.append(attr_name)
300 @docval(*docval_args, allow_positional=AllowPositional.WARNING)
301 def __init__(self, **kwargs):
302 if name is not None: # force container name to be the fixed name in the spec
303 kwargs.update(name=name)
305 # remove arguments from kwargs that correspond to fields that are new (not inherited)
306 # set these arguments after calling base.__init__
307 new_kwargs = dict()
308 for f in new_args:
309 new_kwargs[f] = popargs(f, kwargs) if f in kwargs else None
311 # NOTE: the docval of some constructors do not include all of the fields. the constructor may set
312 # some fields to fixed values. so only keep the kwargs that are used in the constructor docval
313 kwargs_to_pass = {k: v for k, v in kwargs.items() if k in parent_docval_args}
315 base.__init__(self, **kwargs_to_pass) # special case: need to pass self to __init__
316 # TODO should super() be used above instead of base?
318 # set the fields that are new to this class (not inherited)
319 for f, arg_val in new_kwargs.items():
320 setattr(self, f, arg_val)
322 classdict['__init__'] = __init__
325class MCIClassGenerator(CustomClassGenerator):
327 @classmethod
328 def apply_generator_to_field(cls, field_spec, bases, type_map):
329 """Return True if the field spec has quantity * or +, False otherwise."""
330 return getattr(field_spec, 'quantity', None) in (ZERO_OR_MANY, ONE_OR_MANY)
332 @classmethod
333 def process_field_spec(cls, classdict, docval_args, parent_cls, attr_name, not_inherited_fields, type_map, spec):
334 """Add __clsconf__ to the classdict and update the docval args for the field spec with the given attribute name.
335 :param classdict: The dict to update with __clsconf__.
336 :param docval_args: The list of docval arguments.
337 :param parent_cls: The parent class.
338 :param attr_name: The attribute name of the field spec for the container class to generate.
339 :param not_inherited_fields: Dictionary of fields not inherited from the parent class.
340 :param type_map: The type map to use.
341 :param spec: The spec for the container class to generate.
342 """
343 field_spec = not_inherited_fields[attr_name]
344 field_clsconf = dict(
345 attr=attr_name,
346 type=cls._get_type(field_spec, type_map),
347 add='add_{}'.format(attr_name),
348 get='get_{}'.format(attr_name),
349 create='create_{}'.format(attr_name)
350 )
351 classdict.setdefault('__clsconf__', list()).append(field_clsconf)
353 # add a specialized docval arg for __init__
354 docval_arg = dict(
355 name=attr_name,
356 doc=field_spec.doc,
357 type=(list, tuple, dict, cls._get_type(field_spec, type_map))
358 )
359 if cls._check_spec_optional(field_spec, spec):
360 docval_arg['default'] = getattr(field_spec, 'default_value', None)
361 cls._add_to_docval_args(docval_args, docval_arg)
363 @classmethod
364 def post_process(cls, classdict, bases, docval_args, spec):
365 """Add MultiContainerInterface to the list of base classes.
366 :param classdict: The class dictionary.
367 :param bases: The list of base classes.
368 :param docval_args: The dict of docval arguments.
369 :param spec: The spec for the container class to generate.
370 """
371 if '__clsconf__' in classdict:
372 # do not add MCI as a base if a base is already a subclass of MultiContainerInterface
373 for b in bases:
374 if issubclass(b, MultiContainerInterface):
375 break
376 else:
377 if issubclass(MultiContainerInterface, bases[0]):
378 # if bases[0] is Container or another superclass of MCI, then make sure MCI goes first
379 # otherwise, MRO is ambiguous
380 bases.insert(0, MultiContainerInterface)
381 else:
382 # bases[0] is not a subclass of MCI and not a superclass of MCI. place that class first
383 # before MCI. that class __init__ should call super().__init__ which will call the
384 # MCI init
385 bases.insert(1, MultiContainerInterface)
387 @classmethod
388 def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name):
389 if '__clsconf__' in classdict:
390 previous_init = classdict['__init__']
392 @docval(*docval_args, allow_positional=AllowPositional.WARNING)
393 def __init__(self, **kwargs):
394 # store the values passed to init for each MCI attribute so that they can be added
395 # after calling __init__
396 new_kwargs = list()
397 for field_clsconf in classdict['__clsconf__']:
398 attr_name = field_clsconf['attr']
399 # do not store the value if it is None or not present
400 if attr_name not in kwargs or kwargs[attr_name] is None:
401 continue
402 add_method_name = field_clsconf['add']
403 new_kwarg = dict(
404 attr_name=attr_name,
405 value=popargs(attr_name, kwargs),
406 add_method_name=add_method_name
407 )
408 new_kwargs.append(new_kwarg)
410 # pass an empty list to previous_init in case attr_name field is required
411 # (one or many). we do not want previous_init to set the attribute directly.
412 # instead, we will use the add_method after previous_init is finished.
413 kwargs[attr_name] = list()
415 # call the parent class init without the MCI attribute
416 previous_init(self, **kwargs)
418 # call the add method for each MCI attribute
419 for new_kwarg in new_kwargs:
420 add_method = getattr(self, new_kwarg['add_method_name'])
421 add_method(new_kwarg['value'])
423 # override __init__
424 classdict['__init__'] = __init__