Coverage for src/hdmf/build/manager.py: 88%

524 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-04 02:57 +0000

1import logging 

2from collections import OrderedDict, deque 

3from copy import copy 

4 

5from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder 

6from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator 

7from ..container import AbstractContainer, Container, Data 

8from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog 

9from ..spec.spec import BaseStorageSpec 

10from ..utils import docval, getargs, ExtenderMeta, get_docval 

11 

12 

13class Proxy: 

14 """ 

15 A temporary object to represent a Container. This gets used when resolving the true location of a 

16 Container's parent. 

17 Proxy objects allow simple bookkeeping of all potential parents a Container may have. 

18 This object is used by providing all the necessary information for describing the object. This object 

19 gets passed around and candidates are accumulated. Upon calling resolve, all saved candidates are matched 

20 against the information (provided to the constructor). The candidate that has an exact match is returned. 

21 """ 

22 

23 def __init__(self, manager, source, location, namespace, data_type): 

24 self.__source = source 

25 self.__location = location 

26 self.__namespace = namespace 

27 self.__data_type = data_type 

28 self.__manager = manager 

29 self.__candidates = list() 

30 

31 @property 

32 def source(self): 

33 """The source of the object e.g. file source""" 

34 return self.__source 

35 

36 @property 

37 def location(self): 

38 """The location of the object. This can be thought of as a unique path""" 

39 return self.__location 

40 

41 @property 

42 def namespace(self): 

43 """The namespace from which the data_type of this Proxy came from""" 

44 return self.__namespace 

45 

46 @property 

47 def data_type(self): 

48 """The data_type of Container that should match this Proxy""" 

49 return self.__data_type 

50 

51 @docval({"name": "object", "type": (BaseBuilder, Container), "doc": "the container or builder to get a proxy for"}) 

52 def matches(self, **kwargs): 

53 obj = getargs('object', kwargs) 

54 if not isinstance(obj, Proxy): 54 ↛ 56line 54 didn't jump to line 56, because the condition on line 54 was never false

55 obj = self.__manager.get_proxy(obj) 

56 return self == obj 

57 

58 @docval({"name": "container", "type": Container, "doc": "the Container to add as a candidate match"}) 

59 def add_candidate(self, **kwargs): 

60 container = getargs('container', kwargs) 

61 self.__candidates.append(container) 

62 

63 def resolve(self): 

64 for candidate in self.__candidates: 

65 if self.matches(candidate): 

66 return candidate 

67 raise ValueError("No matching candidate Container found for " + self) 

68 

69 def __eq__(self, other): 

70 return self.data_type == other.data_type and \ 

71 self.location == other.location and \ 

72 self.namespace == other.namespace and \ 

73 self.source == other.source 

74 

75 def __repr__(self): 

76 ret = dict() 

77 for key in ('source', 'location', 'namespace', 'data_type'): 

78 ret[key] = getattr(self, key, None) 

79 return str(ret) 

80 

81 

82class BuildManager: 

83 """ 

84 A class for managing builds of AbstractContainers 

85 """ 

86 

87 def __init__(self, type_map): 

88 self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)) 

89 self.__builders = dict() 

90 self.__containers = dict() 

91 self.__active_builders = set() 

92 self.__type_map = type_map 

93 self.__ref_queue = deque() # a queue of the ReferenceBuilders that need to be added 

94 

95 @property 

96 def namespace_catalog(self): 

97 return self.__type_map.namespace_catalog 

98 

99 @property 

100 def type_map(self): 

101 return self.__type_map 

102 

103 @docval({"name": "object", "type": (BaseBuilder, AbstractContainer), 

104 "doc": "the container or builder to get a proxy for"}, 

105 {"name": "source", "type": str, 

106 "doc": "the source of container being built i.e. file path", 'default': None}) 

107 def get_proxy(self, **kwargs): 

108 obj = getargs('object', kwargs) 

109 if isinstance(obj, BaseBuilder): 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true

110 return self._get_proxy_builder(obj) 

111 elif isinstance(obj, AbstractContainer): 111 ↛ exitline 111 didn't return from function 'get_proxy', because the condition on line 111 was never false

112 return self._get_proxy_container(obj) 

113 

114 def _get_proxy_builder(self, builder): 

115 dt = self.__type_map.get_builder_dt(builder) 

116 ns = self.__type_map.get_builder_ns(builder) 

117 stack = list() 

118 tmp = builder 

119 while tmp is not None: 

120 stack.append(tmp.name) 

121 tmp = self.__get_parent_dt_builder(tmp) 

122 loc = "/".join(reversed(stack)) 

123 return Proxy(self, builder.source, loc, ns, dt) 

124 

125 def _get_proxy_container(self, container): 

126 ns, dt = self.__type_map.get_container_ns_dt(container) 

127 stack = list() 

128 tmp = container 

129 while tmp is not None: 

130 if isinstance(tmp, Proxy): 

131 stack.append(tmp.location) 

132 break 

133 else: 

134 stack.append(tmp.name) 

135 tmp = tmp.parent 

136 loc = "/".join(reversed(stack)) 

137 return Proxy(self, container.container_source, loc, ns, dt) 

138 

139 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, 

140 {"name": "source", "type": str, 

141 "doc": "the source of container being built i.e. file path", 'default': None}, 

142 {"name": "spec_ext", "type": BaseStorageSpec, "doc": "a spec that further refines the base specification", 

143 'default': None}, 

144 {"name": "export", "type": bool, "doc": "whether this build is for exporting", 

145 'default': False}, 

146 {"name": "root", "type": bool, "doc": "whether the container is the root of the build process", 

147 'default': False}) 

148 def build(self, **kwargs): 

149 """ Build the GroupBuilder/DatasetBuilder for the given AbstractContainer""" 

150 container, export = getargs('container', 'export', kwargs) 

151 source, spec_ext, root = getargs('source', 'spec_ext', 'root', kwargs) 

152 result = self.get_builder(container) 

153 if root: 

154 self.__active_builders.clear() # reset active builders at start of build process 

155 if result is None: 

156 self.logger.debug("Building new %s '%s' (container_source: %s, source: %s, extended spec: %s, export: %s)" 

157 % (container.__class__.__name__, container.name, repr(container.container_source), 

158 repr(source), spec_ext is not None, export)) 

159 # the container_source is not set or checked when exporting 

160 if not export: 

161 if container.container_source is None: 

162 container.container_source = source 

163 elif source is None: 163 ↛ 164line 163 didn't jump to line 164, because the condition on line 163 was never true

164 source = container.container_source 

165 else: 

166 if container.container_source != source: 166 ↛ 167line 166 didn't jump to line 167, because the condition on line 166 was never true

167 raise ValueError("Cannot change container_source once set: '%s' %s.%s" 

168 % (container.name, container.__class__.__module__, 

169 container.__class__.__name__)) 

170 # NOTE: if exporting, then existing cached builder will be ignored and overridden with new build result 

171 result = self.__type_map.build(container, self, source=source, spec_ext=spec_ext, export=export) 

172 self.prebuilt(container, result) 

173 self.__active_prebuilt(result) 

174 self.logger.debug("Done building %s '%s'" % (container.__class__.__name__, container.name)) 

175 elif not self.__is_active_builder(result) and container.modified: 

176 # if builder was built on file read and is then modified (append mode), it needs to be rebuilt 

177 self.logger.debug("Rebuilding modified %s '%s' (source: %s, extended spec: %s)" 

178 % (container.__class__.__name__, container.name, 

179 repr(source), spec_ext is not None)) 

180 result = self.__type_map.build(container, self, builder=result, source=source, spec_ext=spec_ext, 

181 export=export) 

182 self.logger.debug("Done rebuilding %s '%s'" % (container.__class__.__name__, container.name)) 

183 else: 

184 self.logger.debug("Using prebuilt %s '%s' for %s '%s'" 

185 % (result.__class__.__name__, result.name, 

186 container.__class__.__name__, container.name)) 

187 if root: # create reference builders only after building all other builders 

188 self.__add_refs() 

189 self.__active_builders.clear() # reset active builders now that build process has completed 

190 return result 

191 

192 @docval({"name": "container", "type": AbstractContainer, "doc": "the AbstractContainer to save as prebuilt"}, 

193 {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), 

194 'doc': 'the Builder representation of the given container'}) 

195 def prebuilt(self, **kwargs): 

196 ''' Save the Builder for a given AbstractContainer for future use ''' 

197 container, builder = getargs('container', 'builder', kwargs) 

198 container_id = self.__conthash__(container) 

199 self.__builders[container_id] = builder 

200 builder_id = self.__bldrhash__(builder) 

201 self.__containers[builder_id] = container 

202 

203 def __active_prebuilt(self, builder): 

204 """Save the Builder for future use during the active/current build process.""" 

205 builder_id = self.__bldrhash__(builder) 

206 self.__active_builders.add(builder_id) 

207 

208 def __is_active_builder(self, builder): 

209 """Return True if the Builder was created during the active/current build process.""" 

210 builder_id = self.__bldrhash__(builder) 

211 return builder_id in self.__active_builders 

212 

213 def __conthash__(self, obj): 

214 return id(obj) 

215 

216 def __bldrhash__(self, obj): 

217 return id(obj) 

218 

219 def __add_refs(self): 

220 ''' 

221 Add ReferenceBuilders. 

222 

223 References get queued to be added after all other objects are built. This is because 

224 the current traversal algorithm (i.e. iterating over specs) 

225 does not happen in a guaranteed order. We need to build the targets 

226 of the reference builders so that the targets have the proper parent, 

227 and then write the reference builders after we write everything else. 

228 ''' 

229 while len(self.__ref_queue) > 0: 

230 call = self.__ref_queue.popleft() 

231 self.logger.debug("Adding ReferenceBuilder with call id %d from queue (length %d)" 

232 % (id(call), len(self.__ref_queue))) 

233 call() 

234 

235 def queue_ref(self, func): 

236 '''Set aside creating ReferenceBuilders''' 

237 # TODO: come up with more intelligent way of 

238 # queueing reference resolution, based on reference 

239 # dependency 

240 self.__ref_queue.append(func) 

241 

242 def purge_outdated(self): 

243 containers_copy = self.__containers.copy() 

244 for container in containers_copy.values(): 

245 if container.modified: 

246 container_id = self.__conthash__(container) 

247 builder = self.__builders.get(container_id) 

248 builder_id = self.__bldrhash__(builder) 

249 self.logger.debug("Purging %s '%s' for %s '%s' from prebuilt cache" 

250 % (builder.__class__.__name__, builder.name, 

251 container.__class__.__name__, container.name)) 

252 self.__builders.pop(container_id) 

253 self.__containers.pop(builder_id) 

254 

255 def clear_cache(self): 

256 self.__builders.clear() 

257 self.__containers.clear() 

258 

259 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to get the builder for"}) 

260 def get_builder(self, **kwargs): 

261 """Return the prebuilt builder for the given container or None if it does not exist.""" 

262 container = getargs('container', kwargs) 

263 container_id = self.__conthash__(container) 

264 result = self.__builders.get(container_id) 

265 return result 

266 

267 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), 

268 'doc': 'the builder to construct the AbstractContainer from'}) 

269 def construct(self, **kwargs): 

270 """ Construct the AbstractContainer represented by the given builder """ 

271 builder = getargs('builder', kwargs) 

272 if isinstance(builder, LinkBuilder): 272 ↛ 273line 272 didn't jump to line 273, because the condition on line 272 was never true

273 builder = builder.target 

274 builder_id = self.__bldrhash__(builder) 

275 result = self.__containers.get(builder_id) 

276 if result is None: 

277 parent_builder = self.__get_parent_dt_builder(builder) 

278 if parent_builder is not None: 

279 parent = self._get_proxy_builder(parent_builder) 

280 result = self.__type_map.construct(builder, self, parent) 

281 else: 

282 # we are at the top of the hierarchy, 

283 # so it must be time to resolve parents 

284 result = self.__type_map.construct(builder, self, None) 

285 self.__resolve_parents(result) 

286 self.prebuilt(result, builder) 

287 result.set_modified(False) 

288 return result 

289 

290 def __resolve_parents(self, container): 

291 stack = [container] 

292 while len(stack) > 0: 

293 tmp = stack.pop() 

294 if isinstance(tmp.parent, Proxy): 294 ↛ 295line 294 didn't jump to line 295, because the condition on line 294 was never true

295 tmp.parent = tmp.parent.resolve() 

296 for child in tmp.children: 

297 stack.append(child) 

298 

299 def __get_parent_dt_builder(self, builder): 

300 ''' 

301 Get the next builder above the given builder 

302 that has a data_type 

303 ''' 

304 tmp = builder.parent 

305 ret = None 

306 while tmp is not None: 

307 ret = tmp 

308 dt = self.__type_map.get_builder_dt(tmp) 

309 if dt is not None: 

310 break 

311 tmp = tmp.parent 

312 return ret 

313 

314 # *** The following methods just delegate calls to self.__type_map *** 

315 

316 @docval({'name': 'builder', 'type': Builder, 'doc': 'the Builder to get the class object for'}) 

317 def get_cls(self, **kwargs): 

318 ''' Get the class object for the given Builder ''' 

319 builder = getargs('builder', kwargs) 

320 return self.__type_map.get_cls(builder) 

321 

322 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, 

323 returns='The name a Builder should be given when building this container', rtype=str) 

324 def get_builder_name(self, **kwargs): 

325 ''' Get the name a Builder should be given ''' 

326 container = getargs('container', kwargs) 

327 return self.__type_map.get_builder_name(container) 

328 

329 @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'}, 

330 {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), 

331 'doc': 'the builder to get the sub-specification for'}) 

332 def get_subspec(self, **kwargs): 

333 ''' Get the specification from this spec that corresponds to the given builder ''' 

334 spec, builder = getargs('spec', 'builder', kwargs) 

335 return self.__type_map.get_subspec(spec, builder) 

336 

337 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), 

338 'doc': 'the builder to get the sub-specification for'}) 

339 def get_builder_ns(self, **kwargs): 

340 ''' Get the namespace of a builder ''' 

341 builder = getargs('builder', kwargs) 

342 return self.__type_map.get_builder_ns(builder) 

343 

344 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), 

345 'doc': 'the builder to get the data_type for'}) 

346 def get_builder_dt(self, **kwargs): 

347 ''' 

348 Get the data_type of a builder 

349 ''' 

350 builder = getargs('builder', kwargs) 

351 return self.__type_map.get_builder_dt(builder) 

352 

353 @docval({'name': 'builder', 'type': (GroupBuilder, DatasetBuilder, AbstractContainer), 

354 'doc': 'the builder or container to check'}, 

355 {'name': 'parent_data_type', 'type': str, 

356 'doc': 'the potential parent data_type that refers to a data_type'}, 

357 returns="True if data_type of *builder* is a sub-data_type of *parent_data_type*, False otherwise", 

358 rtype=bool) 

359 def is_sub_data_type(self, **kwargs): 

360 ''' 

361 Return whether or not data_type of *builder* is a sub-data_type of *parent_data_type* 

362 ''' 

363 builder, parent_dt = getargs('builder', 'parent_data_type', kwargs) 

364 if isinstance(builder, (GroupBuilder, DatasetBuilder)): 

365 ns = self.get_builder_ns(builder) 

366 dt = self.get_builder_dt(builder) 

367 else: # builder is an AbstractContainer 

368 ns, dt = self.type_map.get_container_ns_dt(builder) 

369 return self.namespace_catalog.is_sub_data_type(ns, dt, parent_dt) 

370 

371 

372class TypeSource: 

373 '''A class to indicate the source of a data_type in a namespace. 

374 This class should only be used by TypeMap 

375 ''' 

376 

377 @docval({"name": "namespace", "type": str, "doc": "the namespace the from, which the data_type originated"}, 

378 {"name": "data_type", "type": str, "doc": "the name of the type"}) 

379 def __init__(self, **kwargs): 

380 namespace, data_type = getargs('namespace', 'data_type', kwargs) 

381 self.__namespace = namespace 

382 self.__data_type = data_type 

383 

384 @property 

385 def namespace(self): 

386 return self.__namespace 

387 

388 @property 

389 def data_type(self): 

390 return self.__data_type 

391 

392 

393class TypeMap: 

394 ''' A class to maintain the map between ObjectMappers and AbstractContainer classes 

395 ''' 

396 

397 @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None}, 

398 {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}) 

399 def __init__(self, **kwargs): 

400 namespaces, mapper_cls = getargs('namespaces', 'mapper_cls', kwargs) 

401 if namespaces is None: 

402 namespaces = NamespaceCatalog() 

403 if mapper_cls is None: 

404 from .objectmapper import ObjectMapper # avoid circular import 

405 mapper_cls = ObjectMapper 

406 self.__ns_catalog = namespaces 

407 self.__mappers = dict() # already constructed ObjectMapper classes 

408 self.__mapper_cls = dict() # the ObjectMapper class to use for each container type 

409 self.__container_types = OrderedDict() 

410 self.__data_types = dict() 

411 self.__default_mapper_cls = mapper_cls 

412 self.__class_generator = ClassGenerator() 

413 self.register_generator(CustomClassGenerator) 

414 self.register_generator(MCIClassGenerator) 

415 

416 @property 

417 def namespace_catalog(self): 

418 return self.__ns_catalog 

419 

420 @property 

421 def container_types(self): 

422 return self.__container_types 

423 

424 def __copy__(self): 

425 ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls) 

426 ret.merge(self) 

427 return ret 

428 

429 def __deepcopy__(self, memo): 

430 # XXX: From @nicain: All of a sudden legacy tests started 

431 # needing this argument in deepcopy. Doesn't hurt anything, though. 

432 return self.__copy__() 

433 

434 def copy_mappers(self, type_map): 

435 for namespace in self.__ns_catalog.namespaces: 

436 if namespace not in type_map.__container_types: 

437 continue 

438 for data_type in self.__ns_catalog.get_namespace(namespace).get_registered_types(): 

439 container_cls = type_map.__container_types[namespace].get(data_type) 

440 if container_cls is None: 

441 continue 

442 self.register_container_type(namespace, data_type, container_cls) 

443 if container_cls in type_map.__mapper_cls: 

444 self.register_map(container_cls, type_map.__mapper_cls[container_cls]) 

445 

446 def merge(self, type_map, ns_catalog=False): 

447 if ns_catalog: 

448 self.namespace_catalog.merge(type_map.namespace_catalog) 

449 for namespace in type_map.__container_types: 

450 for data_type in type_map.__container_types[namespace]: 

451 container_cls = type_map.__container_types[namespace][data_type] 

452 self.register_container_type(namespace, data_type, container_cls) 

453 for container_cls in type_map.__mapper_cls: 

454 self.register_map(container_cls, type_map.__mapper_cls[container_cls]) 

455 for custom_generators in reversed(type_map.__class_generator.custom_generators): 

456 # iterate in reverse order because generators are stored internally as a stack 

457 self.register_generator(custom_generators) 

458 

459 @docval({"name": "generator", "type": type, "doc": "the CustomClassGenerator class to register"}) 

460 def register_generator(self, **kwargs): 

461 """Add a custom class generator.""" 

462 generator = getargs('generator', kwargs) 

463 self.__class_generator.register_generator(generator) 

464 

465 @docval(*get_docval(NamespaceCatalog.load_namespaces), 

466 returns="the namespaces loaded from the given file", rtype=dict) 

467 def load_namespaces(self, **kwargs): 

468 '''Load namespaces from a namespace file. 

469 This method will call load_namespaces on the NamespaceCatalog used to construct this TypeMap. Additionally, 

470 it will process the return value to keep track of what types were included in the loaded namespaces. Calling 

471 load_namespaces here has the advantage of being able to keep track of type dependencies across namespaces. 

472 ''' 

473 deps = self.__ns_catalog.load_namespaces(**kwargs) 

474 for new_ns, ns_deps in deps.items(): 

475 for src_ns, types in ns_deps.items(): 

476 for dt in types: 

477 container_cls = self.get_dt_container_cls(dt, src_ns, autogen=False) 

478 if container_cls is None: 

479 container_cls = TypeSource(src_ns, dt) 

480 self.register_container_type(new_ns, dt, container_cls) 

481 return deps 

482 

483 @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"}, 

484 {"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, 

485 {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, 

486 returns='the class for the given namespace and data_type', rtype=type) 

487 def get_container_cls(self, **kwargs): 

488 """Get the container class from data type specification. 

489 If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically 

490 created and returned. 

491 """ 

492 # NOTE: this internally used function get_container_cls will be removed in favor of get_dt_container_cls 

493 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) 

494 return self.get_dt_container_cls(data_type, namespace, autogen) 

495 

496 @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, 

497 {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, 

498 {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, 

499 returns='the class for the given namespace and data_type', rtype=type) 

500 def get_dt_container_cls(self, **kwargs): 

501 """Get the container class from data type specification. 

502 If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically 

503 created and returned. 

504 

505 Replaces get_container_cls but namespace is optional. If namespace is unknown, it will be looked up from 

506 all namespaces. 

507 """ 

508 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) 

509 

510 # namespace is unknown, so look it up 

511 if namespace is None: 

512 for ns_key, ns_data_types in self.__container_types.items(): 

513 # NOTE that the type_name may appear in multiple namespaces based on how they were resolved 

514 # but the same type_name should point to the same class 

515 if data_type in ns_data_types: 

516 namespace = ns_key 

517 break 

518 if namespace is None: 

519 raise ValueError("Namespace could not be resolved.") 

520 

521 cls = self.__get_container_cls(namespace, data_type) 

522 if cls is None and autogen: # dynamically generate a class 

523 spec = self.__ns_catalog.get_spec(namespace, data_type) 

524 self.__check_dependent_types(spec, namespace) 

525 parent_cls = self.__get_parent_cls(namespace, data_type, spec) 

526 attr_names = self.__default_mapper_cls.get_attr_names(spec) 

527 cls = self.__class_generator.generate_class(data_type, spec, parent_cls, attr_names, self) 

528 self.register_container_type(namespace, data_type, cls) 

529 return cls 

530 

531 def __check_dependent_types(self, spec, namespace): 

532 """Ensure that classes for all types used by this type exist in this namespace and generate them if not. 

533 """ 

534 def __check_dependent_types_helper(spec, namespace): 

535 if isinstance(spec, (GroupSpec, DatasetSpec)): 535 ↛ 541line 535 didn't jump to line 541, because the condition on line 535 was never false

536 if spec.data_type_inc is not None: 

537 self.get_dt_container_cls(spec.data_type_inc, namespace) # TODO handle recursive definitions 

538 if spec.data_type_def is not None: # nested type definition 538 ↛ 539line 538 didn't jump to line 539, because the condition on line 538 was never true

539 self.get_dt_container_cls(spec.data_type_def, namespace) 

540 else: # spec is a LinkSpec 

541 self.get_dt_container_cls(spec.target_type, namespace) 

542 if isinstance(spec, GroupSpec): 

543 for child_spec in (spec.groups + spec.datasets + spec.links): 543 ↛ 544line 543 didn't jump to line 544, because the loop on line 543 never started

544 __check_dependent_types_helper(child_spec, namespace) 

545 

546 if spec.data_type_inc is not None: 

547 self.get_dt_container_cls(spec.data_type_inc, namespace) 

548 if isinstance(spec, GroupSpec): 

549 for child_spec in (spec.groups + spec.datasets + spec.links): 

550 __check_dependent_types_helper(child_spec, namespace) 

551 

552 def __get_parent_cls(self, namespace, data_type, spec): 

553 dt_hier = self.__ns_catalog.get_hierarchy(namespace, data_type) 

554 dt_hier = dt_hier[1:] # remove the current data_type 

555 parent_cls = None 

556 for t in dt_hier: 

557 parent_cls = self.__get_container_cls(namespace, t) 

558 if parent_cls is not None: 558 ↛ 556line 558 didn't jump to line 556, because the condition on line 558 was never false

559 break 

560 if parent_cls is None: 

561 if isinstance(spec, GroupSpec): 

562 parent_cls = Container 

563 elif isinstance(spec, DatasetSpec): 563 ↛ 566line 563 didn't jump to line 566, because the condition on line 563 was never false

564 parent_cls = Data 

565 else: 

566 raise ValueError("Cannot generate class from %s" % type(spec)) 

567 if type(parent_cls) is not ExtenderMeta: 567 ↛ 568line 567 didn't jump to line 568, because the condition on line 567 was never true

568 raise ValueError("parent class %s is not of type ExtenderMeta - %s" % (parent_cls, type(parent_cls))) 

569 return parent_cls 

570 

571 def __get_container_cls(self, namespace, data_type): 

572 """Get the container class for the namespace, data_type. If the class doesn't exist yet, generate it.""" 

573 if namespace not in self.__container_types: 

574 return None 

575 if data_type not in self.__container_types[namespace]: 

576 return None 

577 ret = self.__container_types[namespace][data_type] 

578 if isinstance(ret, TypeSource): # data_type is a dependency from ret.namespace 

579 cls = self.get_dt_container_cls(ret.data_type, ret.namespace) # get class / generate class 

580 # register the same class into this namespace (replaces TypeSource) 

581 self.register_container_type(namespace, data_type, cls) 

582 ret = cls 

583 return ret 

584 

585 @docval({'name': 'obj', 'type': (GroupBuilder, DatasetBuilder, LinkBuilder, GroupSpec, DatasetSpec), 

586 'doc': 'the object to get the type key for'}) 

587 def __type_key(self, obj): 

588 """ 

589 A wrapper function to simplify the process of getting a type_key for an object. 

590 The type_key is used to get the data_type from a Builder's attributes. 

591 """ 

592 if isinstance(obj, LinkBuilder): 592 ↛ 593line 592 didn't jump to line 593, because the condition on line 592 was never true

593 obj = obj.builder 

594 if isinstance(obj, (GroupBuilder, GroupSpec)): 

595 return self.__ns_catalog.group_spec_cls.type_key() 

596 else: 

597 return self.__ns_catalog.dataset_spec_cls.type_key() 

598 

599 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), 

600 'doc': 'the builder to get the data_type for'}) 

601 def get_builder_dt(self, **kwargs): 

602 ''' 

603 Get the data_type of a builder 

604 ''' 

605 builder = getargs('builder', kwargs) 

606 ret = None 

607 if isinstance(builder, LinkBuilder): 607 ↛ 608line 607 didn't jump to line 608, because the condition on line 607 was never true

608 builder = builder.builder 

609 if isinstance(builder, GroupBuilder): 

610 ret = builder.attributes.get(self.__ns_catalog.group_spec_cls.type_key()) 

611 else: 

612 ret = builder.attributes.get(self.__ns_catalog.dataset_spec_cls.type_key()) 

613 if isinstance(ret, bytes): 613 ↛ 614line 613 didn't jump to line 614, because the condition on line 613 was never true

614 ret = ret.decode('UTF-8') 

615 return ret 

616 

617 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), 

618 'doc': 'the builder to get the sub-specification for'}) 

619 def get_builder_ns(self, **kwargs): 

620 ''' Get the namespace of a builder ''' 

621 builder = getargs('builder', kwargs) 

622 if isinstance(builder, LinkBuilder): 

623 builder = builder.builder 

624 ret = builder.attributes.get('namespace') 

625 return ret 

626 

627 @docval({'name': 'builder', 'type': Builder, 

628 'doc': 'the Builder object to get the corresponding AbstractContainer class for'}) 

629 def get_cls(self, **kwargs): 

630 ''' Get the class object for the given Builder ''' 

631 builder = getargs('builder', kwargs) 

632 data_type = self.get_builder_dt(builder) 

633 if data_type is None: 633 ↛ 634line 633 didn't jump to line 634, because the condition on line 633 was never true

634 raise ValueError("No data_type found for builder %s" % builder.path) 

635 namespace = self.get_builder_ns(builder) 

636 if namespace is None: 636 ↛ 637line 636 didn't jump to line 637, because the condition on line 636 was never true

637 raise ValueError("No namespace found for builder %s" % builder.path) 

638 return self.get_dt_container_cls(data_type, namespace) 

639 

640 @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'}, 

641 {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), 

642 'doc': 'the builder to get the sub-specification for'}) 

643 def get_subspec(self, **kwargs): 

644 ''' Get the specification from this spec that corresponds to the given builder ''' 

645 spec, builder = getargs('spec', 'builder', kwargs) 

646 if isinstance(builder, LinkBuilder): 

647 builder_type = type(builder.builder) 

648 # TODO consider checking against spec.get_link 

649 else: 

650 builder_type = type(builder) 

651 if issubclass(builder_type, DatasetBuilder): 

652 subspec = spec.get_dataset(builder.name) 

653 else: 

654 subspec = spec.get_group(builder.name) 

655 if subspec is None: 

656 # builder was generated from something with a data_type and a wildcard name 

657 if isinstance(builder, LinkBuilder): 

658 dt = self.get_builder_dt(builder.builder) 

659 else: 

660 dt = self.get_builder_dt(builder) 

661 if dt is not None: 

662 ns = self.get_builder_ns(builder) 

663 hierarchy = self.__ns_catalog.get_hierarchy(ns, dt) 

664 for t in hierarchy: 

665 subspec = spec.get_data_type(t) 

666 if subspec is not None: 

667 break 

668 subspec = spec.get_target_type(t) 

669 if subspec is not None: 

670 break 

671 return subspec 

672 

673 def get_container_ns_dt(self, obj): 

674 container_cls = obj.__class__ 

675 namespace, data_type = self.get_container_cls_dt(container_cls) 

676 return namespace, data_type 

677 

678 def get_container_cls_dt(self, cls): 

679 def_ret = (None, None) 

680 for _cls in cls.__mro__: # pragma: no branch 

681 ret = self.__data_types.get(_cls, def_ret) 

682 if ret is not def_ret: 

683 return ret 

684 return ret 

685 

686 @docval({'name': 'namespace', 'type': str, 

687 'doc': 'the namespace to get the container classes for', 'default': None}) 

688 def get_container_classes(self, **kwargs): 

689 namespace = getargs('namespace', kwargs) 

690 ret = self.__data_types.keys() 

691 if namespace is not None: 

692 ret = filter(lambda x: self.__data_types[x][0] == namespace, ret) 

693 return list(ret) 

694 

695 @docval({'name': 'obj', 'type': (AbstractContainer, Builder), 'doc': 'the object to get the ObjectMapper for'}, 

696 returns='the ObjectMapper to use for mapping the given object', rtype='ObjectMapper') 

697 def get_map(self, **kwargs): 

698 """ Return the ObjectMapper object that should be used for the given container """ 

699 obj = getargs('obj', kwargs) 

700 # get the container class, and namespace/data_type 

701 if isinstance(obj, AbstractContainer): 

702 container_cls = obj.__class__ 

703 namespace, data_type = self.get_container_cls_dt(container_cls) 

704 if namespace is None: 704 ↛ 705line 704 didn't jump to line 705, because the condition on line 704 was never true

705 raise ValueError("class %s is not mapped to a data_type" % container_cls) 

706 else: 

707 data_type = self.get_builder_dt(obj) 

708 namespace = self.get_builder_ns(obj) 

709 container_cls = self.get_cls(obj) 

710 # now build the ObjectMapper class 

711 mapper = self.__mappers.get(container_cls) 

712 if mapper is None: 

713 mapper_cls = self.__default_mapper_cls 

714 for cls in container_cls.__mro__: 

715 tmp_mapper_cls = self.__mapper_cls.get(cls) 

716 if tmp_mapper_cls is not None: 

717 mapper_cls = tmp_mapper_cls 

718 break 

719 spec = self.__ns_catalog.get_spec(namespace, data_type) 

720 mapper = mapper_cls(spec) 

721 self.__mappers[container_cls] = mapper 

722 return mapper 

723 

724 @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type to map the class to"}, 

725 {"name": "data_type", "type": str, "doc": "the data_type to map the class to"}, 

726 {"name": "container_cls", "type": (TypeSource, type), "doc": "the class to map to the specified data_type"}) 

727 def register_container_type(self, **kwargs): 

728 ''' Map a container class to a data_type ''' 

729 namespace, data_type, container_cls = getargs('namespace', 'data_type', 'container_cls', kwargs) 

730 spec = self.__ns_catalog.get_spec(namespace, data_type) # make sure the spec exists 

731 self.__container_types.setdefault(namespace, dict()) 

732 self.__container_types[namespace][data_type] = container_cls 

733 self.__data_types.setdefault(container_cls, (namespace, data_type)) 

734 if not isinstance(container_cls, TypeSource): 

735 setattr(container_cls, spec.type_key(), data_type) 

736 setattr(container_cls, 'namespace', namespace) 

737 

738 @docval({"name": "container_cls", "type": type, 

739 "doc": "the AbstractContainer class for which the given ObjectMapper class gets used for"}, 

740 {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map"}) 

741 def register_map(self, **kwargs): 

742 ''' Map a container class to an ObjectMapper class ''' 

743 container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs) 

744 if self.get_container_cls_dt(container_cls) == (None, None): 744 ↛ 745line 744 didn't jump to line 745, because the condition on line 744 was never true

745 raise ValueError('cannot register map for type %s - no data_type found' % container_cls) 

746 self.__mapper_cls[container_cls] = mapper_cls 

747 

748 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, 

749 {"name": "manager", "type": BuildManager, 

750 "doc": "the BuildManager to use for managing this build", 'default': None}, 

751 {"name": "source", "type": str, 

752 "doc": "the source of container being built i.e. file path", 'default': None}, 

753 {"name": "builder", "type": BaseBuilder, "doc": "the Builder to build on", 'default': None}, 

754 {"name": "spec_ext", "type": BaseStorageSpec, "doc": "a spec extension", 'default': None}, 

755 {"name": "export", "type": bool, "doc": "whether this build is for exporting", 

756 'default': False}) 

757 def build(self, **kwargs): 

758 """Build the GroupBuilder/DatasetBuilder for the given AbstractContainer""" 

759 container, manager, builder = getargs('container', 'manager', 'builder', kwargs) 

760 source, spec_ext, export = getargs('source', 'spec_ext', 'export', kwargs) 

761 

762 # get the ObjectMapper to map between Spec objects and AbstractContainer attributes 

763 obj_mapper = self.get_map(container) 

764 if obj_mapper is None: 764 ↛ 765line 764 didn't jump to line 765, because the condition on line 764 was never true

765 raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__)) 

766 

767 # convert the container to a builder using the ObjectMapper 

768 if manager is None: 

769 manager = BuildManager(self) 

770 builder = obj_mapper.build(container, manager, builder=builder, source=source, spec_ext=spec_ext, export=export) 

771 

772 # add additional attributes (namespace, data_type, object_id) to builder 

773 namespace, data_type = self.get_container_ns_dt(container) 

774 builder.set_attribute('namespace', namespace) 

775 builder.set_attribute(self.__type_key(obj_mapper.spec), data_type) 

776 builder.set_attribute(obj_mapper.spec.id_key(), container.object_id) 

777 return builder 

778 

779 @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), 

780 'doc': 'the builder to construct the AbstractContainer from'}, 

781 {'name': 'build_manager', 'type': BuildManager, 

782 'doc': 'the BuildManager for constructing', 'default': None}, 

783 {'name': 'parent', 'type': (Proxy, Container), 

784 'doc': 'the parent Container/Proxy for the Container being built', 'default': None}) 

785 def construct(self, **kwargs): 

786 """ Construct the AbstractContainer represented by the given builder """ 

787 builder, build_manager, parent = getargs('builder', 'build_manager', 'parent', kwargs) 

788 if build_manager is None: 788 ↛ 789line 788 didn't jump to line 789, because the condition on line 788 was never true

789 build_manager = BuildManager(self) 

790 obj_mapper = self.get_map(builder) 

791 if obj_mapper is None: 791 ↛ 792line 791 didn't jump to line 792, because the condition on line 791 was never true

792 dt = builder.attributes[self.namespace_catalog.group_spec_cls.type_key()] 

793 raise ValueError('No ObjectMapper found for builder of type %s' % dt) 

794 else: 

795 return obj_mapper.construct(builder, build_manager, parent) 

796 

797 @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, 

798 returns='The name a Builder should be given when building this container', rtype=str) 

799 def get_builder_name(self, **kwargs): 

800 ''' Get the name a Builder should be given ''' 

801 container = getargs('container', kwargs) 

802 obj_mapper = self.get_map(container) 

803 if obj_mapper is None: 803 ↛ 804line 803 didn't jump to line 804, because the condition on line 803 was never true

804 raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__)) 

805 else: 

806 return obj_mapper.get_builder_name(container)