Coverage for src/hdmf/build/manager.py: 88%
524 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-08-18 20:49 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-08-18 20:49 +0000
1import logging
2from collections import OrderedDict, deque
3from copy import copy
5from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder
6from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator
7from ..container import AbstractContainer, Container, Data
8from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog
9from ..spec.spec import BaseStorageSpec
10from ..utils import docval, getargs, ExtenderMeta, get_docval
13class Proxy:
14 """
15 A temporary object to represent a Container. This gets used when resolving the true location of a
16 Container's parent.
17 Proxy objects allow simple bookkeeping of all potential parents a Container may have.
18 This object is used by providing all the necessary information for describing the object. This object
19 gets passed around and candidates are accumulated. Upon calling resolve, all saved candidates are matched
20 against the information (provided to the constructor). The candidate that has an exact match is returned.
21 """
23 def __init__(self, manager, source, location, namespace, data_type):
24 self.__source = source
25 self.__location = location
26 self.__namespace = namespace
27 self.__data_type = data_type
28 self.__manager = manager
29 self.__candidates = list()
31 @property
32 def source(self):
33 """The source of the object e.g. file source"""
34 return self.__source
36 @property
37 def location(self):
38 """The location of the object. This can be thought of as a unique path"""
39 return self.__location
41 @property
42 def namespace(self):
43 """The namespace from which the data_type of this Proxy came from"""
44 return self.__namespace
46 @property
47 def data_type(self):
48 """The data_type of Container that should match this Proxy"""
49 return self.__data_type
51 @docval({"name": "object", "type": (BaseBuilder, Container), "doc": "the container or builder to get a proxy for"})
52 def matches(self, **kwargs):
53 obj = getargs('object', kwargs)
54 if not isinstance(obj, Proxy): 54 ↛ 56line 54 didn't jump to line 56, because the condition on line 54 was never false
55 obj = self.__manager.get_proxy(obj)
56 return self == obj
58 @docval({"name": "container", "type": Container, "doc": "the Container to add as a candidate match"})
59 def add_candidate(self, **kwargs):
60 container = getargs('container', kwargs)
61 self.__candidates.append(container)
63 def resolve(self):
64 for candidate in self.__candidates:
65 if self.matches(candidate):
66 return candidate
67 raise ValueError("No matching candidate Container found for " + self)
69 def __eq__(self, other):
70 return self.data_type == other.data_type and \
71 self.location == other.location and \
72 self.namespace == other.namespace and \
73 self.source == other.source
75 def __repr__(self):
76 ret = dict()
77 for key in ('source', 'location', 'namespace', 'data_type'):
78 ret[key] = getattr(self, key, None)
79 return str(ret)
82class BuildManager:
83 """
84 A class for managing builds of AbstractContainers
85 """
87 def __init__(self, type_map):
88 self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__))
89 self.__builders = dict()
90 self.__containers = dict()
91 self.__active_builders = set()
92 self.__type_map = type_map
93 self.__ref_queue = deque() # a queue of the ReferenceBuilders that need to be added
95 @property
96 def namespace_catalog(self):
97 return self.__type_map.namespace_catalog
99 @property
100 def type_map(self):
101 return self.__type_map
103 @docval({"name": "object", "type": (BaseBuilder, AbstractContainer),
104 "doc": "the container or builder to get a proxy for"},
105 {"name": "source", "type": str,
106 "doc": "the source of container being built i.e. file path", 'default': None})
107 def get_proxy(self, **kwargs):
108 obj = getargs('object', kwargs)
109 if isinstance(obj, BaseBuilder): 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true
110 return self._get_proxy_builder(obj)
111 elif isinstance(obj, AbstractContainer): 111 ↛ exitline 111 didn't return from function 'get_proxy', because the condition on line 111 was never false
112 return self._get_proxy_container(obj)
114 def _get_proxy_builder(self, builder):
115 dt = self.__type_map.get_builder_dt(builder)
116 ns = self.__type_map.get_builder_ns(builder)
117 stack = list()
118 tmp = builder
119 while tmp is not None:
120 stack.append(tmp.name)
121 tmp = self.__get_parent_dt_builder(tmp)
122 loc = "/".join(reversed(stack))
123 return Proxy(self, builder.source, loc, ns, dt)
125 def _get_proxy_container(self, container):
126 ns, dt = self.__type_map.get_container_ns_dt(container)
127 stack = list()
128 tmp = container
129 while tmp is not None:
130 if isinstance(tmp, Proxy):
131 stack.append(tmp.location)
132 break
133 else:
134 stack.append(tmp.name)
135 tmp = tmp.parent
136 loc = "/".join(reversed(stack))
137 return Proxy(self, container.container_source, loc, ns, dt)
139 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"},
140 {"name": "source", "type": str,
141 "doc": "the source of container being built i.e. file path", 'default': None},
142 {"name": "spec_ext", "type": BaseStorageSpec, "doc": "a spec that further refines the base specification",
143 'default': None},
144 {"name": "export", "type": bool, "doc": "whether this build is for exporting",
145 'default': False},
146 {"name": "root", "type": bool, "doc": "whether the container is the root of the build process",
147 'default': False})
148 def build(self, **kwargs):
149 """ Build the GroupBuilder/DatasetBuilder for the given AbstractContainer"""
150 container, export = getargs('container', 'export', kwargs)
151 source, spec_ext, root = getargs('source', 'spec_ext', 'root', kwargs)
152 result = self.get_builder(container)
153 if root:
154 self.__active_builders.clear() # reset active builders at start of build process
155 if result is None:
156 self.logger.debug("Building new %s '%s' (container_source: %s, source: %s, extended spec: %s, export: %s)"
157 % (container.__class__.__name__, container.name, repr(container.container_source),
158 repr(source), spec_ext is not None, export))
159 # the container_source is not set or checked when exporting
160 if not export:
161 if container.container_source is None:
162 container.container_source = source
163 elif source is None: 163 ↛ 164line 163 didn't jump to line 164, because the condition on line 163 was never true
164 source = container.container_source
165 else:
166 if container.container_source != source: 166 ↛ 167line 166 didn't jump to line 167, because the condition on line 166 was never true
167 raise ValueError("Cannot change container_source once set: '%s' %s.%s"
168 % (container.name, container.__class__.__module__,
169 container.__class__.__name__))
170 # NOTE: if exporting, then existing cached builder will be ignored and overridden with new build result
171 result = self.__type_map.build(container, self, source=source, spec_ext=spec_ext, export=export)
172 self.prebuilt(container, result)
173 self.__active_prebuilt(result)
174 self.logger.debug("Done building %s '%s'" % (container.__class__.__name__, container.name))
175 elif not self.__is_active_builder(result) and container.modified:
176 # if builder was built on file read and is then modified (append mode), it needs to be rebuilt
177 self.logger.debug("Rebuilding modified %s '%s' (source: %s, extended spec: %s)"
178 % (container.__class__.__name__, container.name,
179 repr(source), spec_ext is not None))
180 result = self.__type_map.build(container, self, builder=result, source=source, spec_ext=spec_ext,
181 export=export)
182 self.logger.debug("Done rebuilding %s '%s'" % (container.__class__.__name__, container.name))
183 else:
184 self.logger.debug("Using prebuilt %s '%s' for %s '%s'"
185 % (result.__class__.__name__, result.name,
186 container.__class__.__name__, container.name))
187 if root: # create reference builders only after building all other builders
188 self.__add_refs()
189 self.__active_builders.clear() # reset active builders now that build process has completed
190 return result
192 @docval({"name": "container", "type": AbstractContainer, "doc": "the AbstractContainer to save as prebuilt"},
193 {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder),
194 'doc': 'the Builder representation of the given container'})
195 def prebuilt(self, **kwargs):
196 ''' Save the Builder for a given AbstractContainer for future use '''
197 container, builder = getargs('container', 'builder', kwargs)
198 container_id = self.__conthash__(container)
199 self.__builders[container_id] = builder
200 builder_id = self.__bldrhash__(builder)
201 self.__containers[builder_id] = container
203 def __active_prebuilt(self, builder):
204 """Save the Builder for future use during the active/current build process."""
205 builder_id = self.__bldrhash__(builder)
206 self.__active_builders.add(builder_id)
208 def __is_active_builder(self, builder):
209 """Return True if the Builder was created during the active/current build process."""
210 builder_id = self.__bldrhash__(builder)
211 return builder_id in self.__active_builders
213 def __conthash__(self, obj):
214 return id(obj)
216 def __bldrhash__(self, obj):
217 return id(obj)
219 def __add_refs(self):
220 '''
221 Add ReferenceBuilders.
223 References get queued to be added after all other objects are built. This is because
224 the current traversal algorithm (i.e. iterating over specs)
225 does not happen in a guaranteed order. We need to build the targets
226 of the reference builders so that the targets have the proper parent,
227 and then write the reference builders after we write everything else.
228 '''
229 while len(self.__ref_queue) > 0:
230 call = self.__ref_queue.popleft()
231 self.logger.debug("Adding ReferenceBuilder with call id %d from queue (length %d)"
232 % (id(call), len(self.__ref_queue)))
233 call()
235 def queue_ref(self, func):
236 '''Set aside creating ReferenceBuilders'''
237 # TODO: come up with more intelligent way of
238 # queueing reference resolution, based on reference
239 # dependency
240 self.__ref_queue.append(func)
242 def purge_outdated(self):
243 containers_copy = self.__containers.copy()
244 for container in containers_copy.values():
245 if container.modified:
246 container_id = self.__conthash__(container)
247 builder = self.__builders.get(container_id)
248 builder_id = self.__bldrhash__(builder)
249 self.logger.debug("Purging %s '%s' for %s '%s' from prebuilt cache"
250 % (builder.__class__.__name__, builder.name,
251 container.__class__.__name__, container.name))
252 self.__builders.pop(container_id)
253 self.__containers.pop(builder_id)
255 def clear_cache(self):
256 self.__builders.clear()
257 self.__containers.clear()
259 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to get the builder for"})
260 def get_builder(self, **kwargs):
261 """Return the prebuilt builder for the given container or None if it does not exist."""
262 container = getargs('container', kwargs)
263 container_id = self.__conthash__(container)
264 result = self.__builders.get(container_id)
265 return result
267 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder),
268 'doc': 'the builder to construct the AbstractContainer from'})
269 def construct(self, **kwargs):
270 """ Construct the AbstractContainer represented by the given builder """
271 builder = getargs('builder', kwargs)
272 if isinstance(builder, LinkBuilder): 272 ↛ 273line 272 didn't jump to line 273, because the condition on line 272 was never true
273 builder = builder.target
274 builder_id = self.__bldrhash__(builder)
275 result = self.__containers.get(builder_id)
276 if result is None:
277 parent_builder = self.__get_parent_dt_builder(builder)
278 if parent_builder is not None:
279 parent = self._get_proxy_builder(parent_builder)
280 result = self.__type_map.construct(builder, self, parent)
281 else:
282 # we are at the top of the hierarchy,
283 # so it must be time to resolve parents
284 result = self.__type_map.construct(builder, self, None)
285 self.__resolve_parents(result)
286 self.prebuilt(result, builder)
287 result.set_modified(False)
288 return result
290 def __resolve_parents(self, container):
291 stack = [container]
292 while len(stack) > 0:
293 tmp = stack.pop()
294 if isinstance(tmp.parent, Proxy): 294 ↛ 295line 294 didn't jump to line 295, because the condition on line 294 was never true
295 tmp.parent = tmp.parent.resolve()
296 for child in tmp.children:
297 stack.append(child)
299 def __get_parent_dt_builder(self, builder):
300 '''
301 Get the next builder above the given builder
302 that has a data_type
303 '''
304 tmp = builder.parent
305 ret = None
306 while tmp is not None:
307 ret = tmp
308 dt = self.__type_map.get_builder_dt(tmp)
309 if dt is not None:
310 break
311 tmp = tmp.parent
312 return ret
314 # *** The following methods just delegate calls to self.__type_map ***
316 @docval({'name': 'builder', 'type': Builder, 'doc': 'the Builder to get the class object for'})
317 def get_cls(self, **kwargs):
318 ''' Get the class object for the given Builder '''
319 builder = getargs('builder', kwargs)
320 return self.__type_map.get_cls(builder)
322 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"},
323 returns='The name a Builder should be given when building this container', rtype=str)
324 def get_builder_name(self, **kwargs):
325 ''' Get the name a Builder should be given '''
326 container = getargs('container', kwargs)
327 return self.__type_map.get_builder_name(container)
329 @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'},
330 {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
331 'doc': 'the builder to get the sub-specification for'})
332 def get_subspec(self, **kwargs):
333 ''' Get the specification from this spec that corresponds to the given builder '''
334 spec, builder = getargs('spec', 'builder', kwargs)
335 return self.__type_map.get_subspec(spec, builder)
337 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
338 'doc': 'the builder to get the sub-specification for'})
339 def get_builder_ns(self, **kwargs):
340 ''' Get the namespace of a builder '''
341 builder = getargs('builder', kwargs)
342 return self.__type_map.get_builder_ns(builder)
344 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
345 'doc': 'the builder to get the data_type for'})
346 def get_builder_dt(self, **kwargs):
347 '''
348 Get the data_type of a builder
349 '''
350 builder = getargs('builder', kwargs)
351 return self.__type_map.get_builder_dt(builder)
353 @docval({'name': 'builder', 'type': (GroupBuilder, DatasetBuilder, AbstractContainer),
354 'doc': 'the builder or container to check'},
355 {'name': 'parent_data_type', 'type': str,
356 'doc': 'the potential parent data_type that refers to a data_type'},
357 returns="True if data_type of *builder* is a sub-data_type of *parent_data_type*, False otherwise",
358 rtype=bool)
359 def is_sub_data_type(self, **kwargs):
360 '''
361 Return whether or not data_type of *builder* is a sub-data_type of *parent_data_type*
362 '''
363 builder, parent_dt = getargs('builder', 'parent_data_type', kwargs)
364 if isinstance(builder, (GroupBuilder, DatasetBuilder)):
365 ns = self.get_builder_ns(builder)
366 dt = self.get_builder_dt(builder)
367 else: # builder is an AbstractContainer
368 ns, dt = self.type_map.get_container_ns_dt(builder)
369 return self.namespace_catalog.is_sub_data_type(ns, dt, parent_dt)
372class TypeSource:
373 '''A class to indicate the source of a data_type in a namespace.
374 This class should only be used by TypeMap
375 '''
377 @docval({"name": "namespace", "type": str, "doc": "the namespace the from, which the data_type originated"},
378 {"name": "data_type", "type": str, "doc": "the name of the type"})
379 def __init__(self, **kwargs):
380 namespace, data_type = getargs('namespace', 'data_type', kwargs)
381 self.__namespace = namespace
382 self.__data_type = data_type
384 @property
385 def namespace(self):
386 return self.__namespace
388 @property
389 def data_type(self):
390 return self.__data_type
393class TypeMap:
394 ''' A class to maintain the map between ObjectMappers and AbstractContainer classes
395 '''
397 @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None},
398 {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None})
399 def __init__(self, **kwargs):
400 namespaces, mapper_cls = getargs('namespaces', 'mapper_cls', kwargs)
401 if namespaces is None:
402 namespaces = NamespaceCatalog()
403 if mapper_cls is None:
404 from .objectmapper import ObjectMapper # avoid circular import
405 mapper_cls = ObjectMapper
406 self.__ns_catalog = namespaces
407 self.__mappers = dict() # already constructed ObjectMapper classes
408 self.__mapper_cls = dict() # the ObjectMapper class to use for each container type
409 self.__container_types = OrderedDict()
410 self.__data_types = dict()
411 self.__default_mapper_cls = mapper_cls
412 self.__class_generator = ClassGenerator()
413 self.register_generator(CustomClassGenerator)
414 self.register_generator(MCIClassGenerator)
416 @property
417 def namespace_catalog(self):
418 return self.__ns_catalog
420 @property
421 def container_types(self):
422 return self.__container_types
424 def __copy__(self):
425 ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls)
426 ret.merge(self)
427 return ret
429 def __deepcopy__(self, memo):
430 # XXX: From @nicain: All of a sudden legacy tests started
431 # needing this argument in deepcopy. Doesn't hurt anything, though.
432 return self.__copy__()
434 def copy_mappers(self, type_map):
435 for namespace in self.__ns_catalog.namespaces:
436 if namespace not in type_map.__container_types:
437 continue
438 for data_type in self.__ns_catalog.get_namespace(namespace).get_registered_types():
439 container_cls = type_map.__container_types[namespace].get(data_type)
440 if container_cls is None:
441 continue
442 self.register_container_type(namespace, data_type, container_cls)
443 if container_cls in type_map.__mapper_cls:
444 self.register_map(container_cls, type_map.__mapper_cls[container_cls])
446 def merge(self, type_map, ns_catalog=False):
447 if ns_catalog:
448 self.namespace_catalog.merge(type_map.namespace_catalog)
449 for namespace in type_map.__container_types:
450 for data_type in type_map.__container_types[namespace]:
451 container_cls = type_map.__container_types[namespace][data_type]
452 self.register_container_type(namespace, data_type, container_cls)
453 for container_cls in type_map.__mapper_cls:
454 self.register_map(container_cls, type_map.__mapper_cls[container_cls])
455 for custom_generators in reversed(type_map.__class_generator.custom_generators):
456 # iterate in reverse order because generators are stored internally as a stack
457 self.register_generator(custom_generators)
459 @docval({"name": "generator", "type": type, "doc": "the CustomClassGenerator class to register"})
460 def register_generator(self, **kwargs):
461 """Add a custom class generator."""
462 generator = getargs('generator', kwargs)
463 self.__class_generator.register_generator(generator)
465 @docval(*get_docval(NamespaceCatalog.load_namespaces),
466 returns="the namespaces loaded from the given file", rtype=dict)
467 def load_namespaces(self, **kwargs):
468 '''Load namespaces from a namespace file.
469 This method will call load_namespaces on the NamespaceCatalog used to construct this TypeMap. Additionally,
470 it will process the return value to keep track of what types were included in the loaded namespaces. Calling
471 load_namespaces here has the advantage of being able to keep track of type dependencies across namespaces.
472 '''
473 deps = self.__ns_catalog.load_namespaces(**kwargs)
474 for new_ns, ns_deps in deps.items():
475 for src_ns, types in ns_deps.items():
476 for dt in types:
477 container_cls = self.get_dt_container_cls(dt, src_ns, autogen=False)
478 if container_cls is None:
479 container_cls = TypeSource(src_ns, dt)
480 self.register_container_type(new_ns, dt, container_cls)
481 return deps
483 @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"},
484 {"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"},
485 {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True},
486 returns='the class for the given namespace and data_type', rtype=type)
487 def get_container_cls(self, **kwargs):
488 """Get the container class from data type specification.
489 If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically
490 created and returned.
491 """
492 # NOTE: this internally used function get_container_cls will be removed in favor of get_dt_container_cls
493 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs)
494 return self.get_dt_container_cls(data_type, namespace, autogen)
496 @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"},
497 {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None},
498 {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True},
499 returns='the class for the given namespace and data_type', rtype=type)
500 def get_dt_container_cls(self, **kwargs):
501 """Get the container class from data type specification.
502 If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically
503 created and returned.
505 Replaces get_container_cls but namespace is optional. If namespace is unknown, it will be looked up from
506 all namespaces.
507 """
508 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs)
510 # namespace is unknown, so look it up
511 if namespace is None:
512 for ns_key, ns_data_types in self.__container_types.items():
513 # NOTE that the type_name may appear in multiple namespaces based on how they were resolved
514 # but the same type_name should point to the same class
515 if data_type in ns_data_types:
516 namespace = ns_key
517 break
518 if namespace is None:
519 raise ValueError("Namespace could not be resolved.")
521 cls = self.__get_container_cls(namespace, data_type)
522 if cls is None and autogen: # dynamically generate a class
523 spec = self.__ns_catalog.get_spec(namespace, data_type)
524 self.__check_dependent_types(spec, namespace)
525 parent_cls = self.__get_parent_cls(namespace, data_type, spec)
526 attr_names = self.__default_mapper_cls.get_attr_names(spec)
527 cls = self.__class_generator.generate_class(data_type, spec, parent_cls, attr_names, self)
528 self.register_container_type(namespace, data_type, cls)
529 return cls
531 def __check_dependent_types(self, spec, namespace):
532 """Ensure that classes for all types used by this type exist in this namespace and generate them if not.
533 """
534 def __check_dependent_types_helper(spec, namespace):
535 if isinstance(spec, (GroupSpec, DatasetSpec)): 535 ↛ 541line 535 didn't jump to line 541, because the condition on line 535 was never false
536 if spec.data_type_inc is not None:
537 self.get_dt_container_cls(spec.data_type_inc, namespace) # TODO handle recursive definitions
538 if spec.data_type_def is not None: # nested type definition 538 ↛ 539line 538 didn't jump to line 539, because the condition on line 538 was never true
539 self.get_dt_container_cls(spec.data_type_def, namespace)
540 else: # spec is a LinkSpec
541 self.get_dt_container_cls(spec.target_type, namespace)
542 if isinstance(spec, GroupSpec):
543 for child_spec in (spec.groups + spec.datasets + spec.links): 543 ↛ 544line 543 didn't jump to line 544, because the loop on line 543 never started
544 __check_dependent_types_helper(child_spec, namespace)
546 if spec.data_type_inc is not None:
547 self.get_dt_container_cls(spec.data_type_inc, namespace)
548 if isinstance(spec, GroupSpec):
549 for child_spec in (spec.groups + spec.datasets + spec.links):
550 __check_dependent_types_helper(child_spec, namespace)
552 def __get_parent_cls(self, namespace, data_type, spec):
553 dt_hier = self.__ns_catalog.get_hierarchy(namespace, data_type)
554 dt_hier = dt_hier[1:] # remove the current data_type
555 parent_cls = None
556 for t in dt_hier:
557 parent_cls = self.__get_container_cls(namespace, t)
558 if parent_cls is not None: 558 ↛ 556line 558 didn't jump to line 556, because the condition on line 558 was never false
559 break
560 if parent_cls is None:
561 if isinstance(spec, GroupSpec):
562 parent_cls = Container
563 elif isinstance(spec, DatasetSpec): 563 ↛ 566line 563 didn't jump to line 566, because the condition on line 563 was never false
564 parent_cls = Data
565 else:
566 raise ValueError("Cannot generate class from %s" % type(spec))
567 if type(parent_cls) is not ExtenderMeta: 567 ↛ 568line 567 didn't jump to line 568, because the condition on line 567 was never true
568 raise ValueError("parent class %s is not of type ExtenderMeta - %s" % (parent_cls, type(parent_cls)))
569 return parent_cls
571 def __get_container_cls(self, namespace, data_type):
572 """Get the container class for the namespace, data_type. If the class doesn't exist yet, generate it."""
573 if namespace not in self.__container_types:
574 return None
575 if data_type not in self.__container_types[namespace]:
576 return None
577 ret = self.__container_types[namespace][data_type]
578 if isinstance(ret, TypeSource): # data_type is a dependency from ret.namespace
579 cls = self.get_dt_container_cls(ret.data_type, ret.namespace) # get class / generate class
580 # register the same class into this namespace (replaces TypeSource)
581 self.register_container_type(namespace, data_type, cls)
582 ret = cls
583 return ret
585 @docval({'name': 'obj', 'type': (GroupBuilder, DatasetBuilder, LinkBuilder, GroupSpec, DatasetSpec),
586 'doc': 'the object to get the type key for'})
587 def __type_key(self, obj):
588 """
589 A wrapper function to simplify the process of getting a type_key for an object.
590 The type_key is used to get the data_type from a Builder's attributes.
591 """
592 if isinstance(obj, LinkBuilder): 592 ↛ 593line 592 didn't jump to line 593, because the condition on line 592 was never true
593 obj = obj.builder
594 if isinstance(obj, (GroupBuilder, GroupSpec)):
595 return self.__ns_catalog.group_spec_cls.type_key()
596 else:
597 return self.__ns_catalog.dataset_spec_cls.type_key()
599 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
600 'doc': 'the builder to get the data_type for'})
601 def get_builder_dt(self, **kwargs):
602 '''
603 Get the data_type of a builder
604 '''
605 builder = getargs('builder', kwargs)
606 ret = None
607 if isinstance(builder, LinkBuilder): 607 ↛ 608line 607 didn't jump to line 608, because the condition on line 607 was never true
608 builder = builder.builder
609 if isinstance(builder, GroupBuilder):
610 ret = builder.attributes.get(self.__ns_catalog.group_spec_cls.type_key())
611 else:
612 ret = builder.attributes.get(self.__ns_catalog.dataset_spec_cls.type_key())
613 if isinstance(ret, bytes): 613 ↛ 614line 613 didn't jump to line 614, because the condition on line 613 was never true
614 ret = ret.decode('UTF-8')
615 return ret
617 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
618 'doc': 'the builder to get the sub-specification for'})
619 def get_builder_ns(self, **kwargs):
620 ''' Get the namespace of a builder '''
621 builder = getargs('builder', kwargs)
622 if isinstance(builder, LinkBuilder):
623 builder = builder.builder
624 ret = builder.attributes.get('namespace')
625 return ret
627 @docval({'name': 'builder', 'type': Builder,
628 'doc': 'the Builder object to get the corresponding AbstractContainer class for'})
629 def get_cls(self, **kwargs):
630 ''' Get the class object for the given Builder '''
631 builder = getargs('builder', kwargs)
632 data_type = self.get_builder_dt(builder)
633 if data_type is None: 633 ↛ 634line 633 didn't jump to line 634, because the condition on line 633 was never true
634 raise ValueError("No data_type found for builder %s" % builder.path)
635 namespace = self.get_builder_ns(builder)
636 if namespace is None: 636 ↛ 637line 636 didn't jump to line 637, because the condition on line 636 was never true
637 raise ValueError("No namespace found for builder %s" % builder.path)
638 return self.get_dt_container_cls(data_type, namespace)
640 @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'},
641 {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
642 'doc': 'the builder to get the sub-specification for'})
643 def get_subspec(self, **kwargs):
644 ''' Get the specification from this spec that corresponds to the given builder '''
645 spec, builder = getargs('spec', 'builder', kwargs)
646 if isinstance(builder, LinkBuilder):
647 builder_type = type(builder.builder)
648 # TODO consider checking against spec.get_link
649 else:
650 builder_type = type(builder)
651 if issubclass(builder_type, DatasetBuilder):
652 subspec = spec.get_dataset(builder.name)
653 else:
654 subspec = spec.get_group(builder.name)
655 if subspec is None:
656 # builder was generated from something with a data_type and a wildcard name
657 if isinstance(builder, LinkBuilder):
658 dt = self.get_builder_dt(builder.builder)
659 else:
660 dt = self.get_builder_dt(builder)
661 if dt is not None:
662 ns = self.get_builder_ns(builder)
663 hierarchy = self.__ns_catalog.get_hierarchy(ns, dt)
664 for t in hierarchy:
665 subspec = spec.get_data_type(t)
666 if subspec is not None:
667 break
668 subspec = spec.get_target_type(t)
669 if subspec is not None:
670 break
671 return subspec
673 def get_container_ns_dt(self, obj):
674 container_cls = obj.__class__
675 namespace, data_type = self.get_container_cls_dt(container_cls)
676 return namespace, data_type
678 def get_container_cls_dt(self, cls):
679 def_ret = (None, None)
680 for _cls in cls.__mro__: # pragma: no branch
681 ret = self.__data_types.get(_cls, def_ret)
682 if ret is not def_ret:
683 return ret
684 return ret
686 @docval({'name': 'namespace', 'type': str,
687 'doc': 'the namespace to get the container classes for', 'default': None})
688 def get_container_classes(self, **kwargs):
689 namespace = getargs('namespace', kwargs)
690 ret = self.__data_types.keys()
691 if namespace is not None:
692 ret = filter(lambda x: self.__data_types[x][0] == namespace, ret)
693 return list(ret)
695 @docval({'name': 'obj', 'type': (AbstractContainer, Builder), 'doc': 'the object to get the ObjectMapper for'},
696 returns='the ObjectMapper to use for mapping the given object', rtype='ObjectMapper')
697 def get_map(self, **kwargs):
698 """ Return the ObjectMapper object that should be used for the given container """
699 obj = getargs('obj', kwargs)
700 # get the container class, and namespace/data_type
701 if isinstance(obj, AbstractContainer):
702 container_cls = obj.__class__
703 namespace, data_type = self.get_container_cls_dt(container_cls)
704 if namespace is None: 704 ↛ 705line 704 didn't jump to line 705, because the condition on line 704 was never true
705 raise ValueError("class %s is not mapped to a data_type" % container_cls)
706 else:
707 data_type = self.get_builder_dt(obj)
708 namespace = self.get_builder_ns(obj)
709 container_cls = self.get_cls(obj)
710 # now build the ObjectMapper class
711 mapper = self.__mappers.get(container_cls)
712 if mapper is None:
713 mapper_cls = self.__default_mapper_cls
714 for cls in container_cls.__mro__:
715 tmp_mapper_cls = self.__mapper_cls.get(cls)
716 if tmp_mapper_cls is not None:
717 mapper_cls = tmp_mapper_cls
718 break
719 spec = self.__ns_catalog.get_spec(namespace, data_type)
720 mapper = mapper_cls(spec)
721 self.__mappers[container_cls] = mapper
722 return mapper
724 @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type to map the class to"},
725 {"name": "data_type", "type": str, "doc": "the data_type to map the class to"},
726 {"name": "container_cls", "type": (TypeSource, type), "doc": "the class to map to the specified data_type"})
727 def register_container_type(self, **kwargs):
728 ''' Map a container class to a data_type '''
729 namespace, data_type, container_cls = getargs('namespace', 'data_type', 'container_cls', kwargs)
730 spec = self.__ns_catalog.get_spec(namespace, data_type) # make sure the spec exists
731 self.__container_types.setdefault(namespace, dict())
732 self.__container_types[namespace][data_type] = container_cls
733 self.__data_types.setdefault(container_cls, (namespace, data_type))
734 if not isinstance(container_cls, TypeSource):
735 setattr(container_cls, spec.type_key(), data_type)
736 setattr(container_cls, 'namespace', namespace)
738 @docval({"name": "container_cls", "type": type,
739 "doc": "the AbstractContainer class for which the given ObjectMapper class gets used for"},
740 {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map"})
741 def register_map(self, **kwargs):
742 ''' Map a container class to an ObjectMapper class '''
743 container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs)
744 if self.get_container_cls_dt(container_cls) == (None, None): 744 ↛ 745line 744 didn't jump to line 745, because the condition on line 744 was never true
745 raise ValueError('cannot register map for type %s - no data_type found' % container_cls)
746 self.__mapper_cls[container_cls] = mapper_cls
748 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"},
749 {"name": "manager", "type": BuildManager,
750 "doc": "the BuildManager to use for managing this build", 'default': None},
751 {"name": "source", "type": str,
752 "doc": "the source of container being built i.e. file path", 'default': None},
753 {"name": "builder", "type": BaseBuilder, "doc": "the Builder to build on", 'default': None},
754 {"name": "spec_ext", "type": BaseStorageSpec, "doc": "a spec extension", 'default': None},
755 {"name": "export", "type": bool, "doc": "whether this build is for exporting",
756 'default': False})
757 def build(self, **kwargs):
758 """Build the GroupBuilder/DatasetBuilder for the given AbstractContainer"""
759 container, manager, builder = getargs('container', 'manager', 'builder', kwargs)
760 source, spec_ext, export = getargs('source', 'spec_ext', 'export', kwargs)
762 # get the ObjectMapper to map between Spec objects and AbstractContainer attributes
763 obj_mapper = self.get_map(container)
764 if obj_mapper is None: 764 ↛ 765line 764 didn't jump to line 765, because the condition on line 764 was never true
765 raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__))
767 # convert the container to a builder using the ObjectMapper
768 if manager is None:
769 manager = BuildManager(self)
770 builder = obj_mapper.build(container, manager, builder=builder, source=source, spec_ext=spec_ext, export=export)
772 # add additional attributes (namespace, data_type, object_id) to builder
773 namespace, data_type = self.get_container_ns_dt(container)
774 builder.set_attribute('namespace', namespace)
775 builder.set_attribute(self.__type_key(obj_mapper.spec), data_type)
776 builder.set_attribute(obj_mapper.spec.id_key(), container.object_id)
777 return builder
779 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder),
780 'doc': 'the builder to construct the AbstractContainer from'},
781 {'name': 'build_manager', 'type': BuildManager,
782 'doc': 'the BuildManager for constructing', 'default': None},
783 {'name': 'parent', 'type': (Proxy, Container),
784 'doc': 'the parent Container/Proxy for the Container being built', 'default': None})
785 def construct(self, **kwargs):
786 """ Construct the AbstractContainer represented by the given builder """
787 builder, build_manager, parent = getargs('builder', 'build_manager', 'parent', kwargs)
788 if build_manager is None: 788 ↛ 789line 788 didn't jump to line 789, because the condition on line 788 was never true
789 build_manager = BuildManager(self)
790 obj_mapper = self.get_map(builder)
791 if obj_mapper is None: 791 ↛ 792line 791 didn't jump to line 792, because the condition on line 791 was never true
792 dt = builder.attributes[self.namespace_catalog.group_spec_cls.type_key()]
793 raise ValueError('No ObjectMapper found for builder of type %s' % dt)
794 else:
795 return obj_mapper.construct(builder, build_manager, parent)
797 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"},
798 returns='The name a Builder should be given when building this container', rtype=str)
799 def get_builder_name(self, **kwargs):
800 ''' Get the name a Builder should be given '''
801 container = getargs('container', kwargs)
802 obj_mapper = self.get_map(container)
803 if obj_mapper is None: 803 ↛ 804line 803 didn't jump to line 804, because the condition on line 803 was never true
804 raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__))
805 else:
806 return obj_mapper.get_builder_name(container)