Coverage for src/hdmf/build/classgenerator.py: 97%

199 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-08-18 20:49 +0000

1from copy import deepcopy 

2from datetime import datetime, date 

3 

4import numpy as np 

5 

6from ..container import Container, Data, DataRegion, MultiContainerInterface 

7from ..spec import AttributeSpec, LinkSpec, RefSpec, GroupSpec 

8from ..spec.spec import BaseStorageSpec, ZERO_OR_MANY, ONE_OR_MANY 

9from ..utils import docval, getargs, ExtenderMeta, get_docval, popargs, AllowPositional 

10 

11 

12class ClassGenerator: 

13 

14 def __init__(self): 

15 self.__custom_generators = [] 

16 

17 @property 

18 def custom_generators(self): 

19 return self.__custom_generators 

20 

21 @docval({'name': 'generator', 'type': type, 'doc': 'the CustomClassGenerator class to register'}) 

22 def register_generator(self, **kwargs): 

23 """Add a custom class generator to this ClassGenerator. 

24 

25 Generators added later are run first. Duplicates are moved to the top of the list. 

26 """ 

27 generator = getargs('generator', kwargs) 

28 if not issubclass(generator, CustomClassGenerator): 

29 raise ValueError('Generator %s must be a subclass of CustomClassGenerator.' % generator) 

30 if generator in self.__custom_generators: 

31 self.__custom_generators.remove(generator) 

32 self.__custom_generators.insert(0, generator) 

33 

34 @docval({'name': 'data_type', 'type': str, 'doc': 'the data type to create a AbstractContainer class for'}, 

35 {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, 

36 {'name': 'parent_cls', 'type': type, 'doc': ''}, 

37 {'name': 'attr_names', 'type': dict, 'doc': ''}, 

38 {'name': 'type_map', 'type': 'TypeMap', 'doc': ''}, 

39 returns='the class for the given namespace and data_type', rtype=type) 

40 def generate_class(self, **kwargs): 

41 """Get the container class from data type specification. 

42 If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically 

43 created and returned. 

44 """ 

45 data_type, spec, parent_cls, attr_names, type_map = getargs('data_type', 'spec', 'parent_cls', 'attr_names', 

46 'type_map', kwargs) 

47 

48 not_inherited_fields = dict() 

49 for k, field_spec in attr_names.items(): 

50 if k == 'help': # pragma: no cover 

51 # (legacy) do not add field named 'help' to any part of class object 

52 continue 

53 if isinstance(field_spec, GroupSpec) and field_spec.data_type is None: # skip named, untyped groups 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 continue 

55 if not spec.is_inherited_spec(field_spec): 

56 not_inherited_fields[k] = field_spec 

57 try: 

58 classdict = dict() 

59 bases = [parent_cls] 

60 docval_args = list(deepcopy(get_docval(parent_cls.__init__))) 

61 for attr_name, field_spec in not_inherited_fields.items(): 

62 for class_generator in self.__custom_generators: # pragma: no branch 

63 # each generator can update classdict and docval_args 

64 if class_generator.apply_generator_to_field(field_spec, bases, type_map): 

65 class_generator.process_field_spec(classdict, docval_args, parent_cls, attr_name, 

66 not_inherited_fields, type_map, spec) 

67 break # each field_spec should be processed by only one generator 

68 

69 for class_generator in self.__custom_generators: 

70 class_generator.post_process(classdict, bases, docval_args, spec) 

71 

72 for class_generator in reversed(self.__custom_generators): 

73 # go in reverse order so that base init is added first and 

74 # later class generators can modify or overwrite __init__ set by an earlier class generator 

75 class_generator.set_init(classdict, bases, docval_args, not_inherited_fields, spec.name) 

76 except TypeDoesNotExistError as e: # pragma: no cover 

77 # this error should never happen after hdmf#322 

78 name = spec.data_type_def 

79 if name is None: 

80 name = 'Unknown' 

81 raise ValueError("Cannot dynamically generate class for type '%s'. " % name 

82 + str(e) 

83 + " Please define that type before defining '%s'." % name) 

84 cls = ExtenderMeta(data_type, tuple(bases), classdict) 

85 return cls 

86 

87 

88class TypeDoesNotExistError(Exception): # pragma: no cover 

89 pass 

90 

91 

92class CustomClassGenerator: 

93 """Subclass this class and register an instance to alter how classes are auto-generated.""" 

94 

95 def __new__(cls, *args, **kwargs): # pragma: no cover 

96 raise TypeError('Cannot instantiate class %s' % cls.__name__) 

97 

98 # mapping from spec types to allowable python types for docval for fields during dynamic class generation 

99 # e.g., if a dataset/attribute spec has dtype int32, then get_class should generate a docval for the class' 

100 # __init__ method that allows the types (int, np.int32, np.int64) for the corresponding field. 

101 # passing an np.int16 would raise a docval error. 

102 # passing an int64 to __init__ would result in the field storing the value as an int64 (and subsequently written 

103 # as an int64). no upconversion or downconversion happens as a result of this map 

104 _spec_dtype_map = { 

105 'float32': (float, np.float32, np.float64), 

106 'float': (float, np.float32, np.float64), 

107 'float64': (float, np.float64), 

108 'double': (float, np.float64), 

109 'int8': (np.int8, np.int16, np.int32, np.int64, int), 

110 'int16': (np.int16, np.int32, np.int64, int), 

111 'short': (np.int16, np.int32, np.int64, int), 

112 'int32': (int, np.int32, np.int64), 

113 'int': (int, np.int32, np.int64), 

114 'int64': np.int64, 

115 'long': np.int64, 

116 'uint8': (np.uint8, np.uint16, np.uint32, np.uint64), 

117 'uint16': (np.uint16, np.uint32, np.uint64), 

118 'uint32': (np.uint32, np.uint64), 

119 'uint64': np.uint64, 

120 'numeric': (float, np.float32, np.float64, np.int8, np.int16, np.int32, np.int64, int, np.uint8, np.uint16, 

121 np.uint32, np.uint64), 

122 'text': str, 

123 'utf': str, 

124 'utf8': str, 

125 'utf-8': str, 

126 'ascii': bytes, 

127 'bytes': bytes, 

128 'bool': (bool, np.bool_), 

129 'isodatetime': (datetime, date), 

130 'datetime': (datetime, date) 

131 } 

132 

133 @classmethod 

134 def _get_type_from_spec_dtype(cls, spec_dtype): 

135 """Get the Python type associated with the given spec dtype string. 

136 Raises ValueError if the given dtype has no mapping to a Python type. 

137 """ 

138 dtype = cls._spec_dtype_map.get(spec_dtype) 

139 if dtype is None: # pragma: no cover 

140 # this should not happen as long as _spec_dtype_map is kept up to date with 

141 # hdmf.spec.spec.DtypeHelper.valid_primary_dtypes 

142 raise ValueError("Spec dtype '%s' cannot be mapped to a Python type." % spec_dtype) 

143 return dtype 

144 

145 @classmethod 

146 def _get_container_type(cls, type_name, type_map): 

147 """Search all namespaces for the container class associated with the given data type. 

148 Raises TypeDoesNotExistError if type is not found in any namespace. 

149 """ 

150 container_type = type_map.get_dt_container_cls(type_name) 

151 if container_type is None: # pragma: no cover 

152 # this should never happen after hdmf#322 

153 raise TypeDoesNotExistError("Type '%s' does not exist." % type_name) 

154 return container_type 

155 

156 @classmethod 

157 def _get_type(cls, spec, type_map): 

158 """Get the type of a spec for use in docval. 

159 Returns a container class, a type, a tuple of types, ('array_data', 'data') for specs with 

160 non-scalar shape, or (Data, Container) when an attribute reference target has not been mapped to a container 

161 class. 

162 """ 

163 if isinstance(spec, AttributeSpec): 

164 if isinstance(spec.dtype, RefSpec): 164 ↛ 165line 164 didn't jump to line 165, because the condition on line 164 was never true

165 try: 

166 container_type = cls._get_container_type(spec.dtype.target_type, type_map) 

167 return container_type 

168 except TypeDoesNotExistError: 

169 # TODO what happens when the attribute ref target is not (or not yet) mapped to a container class? 

170 # returning Data, Container works as a generic fallback for now but should be more specific 

171 return Data, Container 

172 elif spec.shape is None and spec.dims is None: 

173 return cls._get_type_from_spec_dtype(spec.dtype) 

174 else: 

175 return 'array_data', 'data' 

176 if isinstance(spec, LinkSpec): 

177 return cls._get_container_type(spec.target_type, type_map) 

178 if spec.data_type is not None: 

179 return cls._get_container_type(spec.data_type, type_map) 

180 if spec.shape is None and spec.dims is None: 

181 return cls._get_type_from_spec_dtype(spec.dtype) 

182 return 'array_data', 'data' 

183 

184 @classmethod 

185 def _ischild(cls, dtype): 

186 """Check if dtype represents a type that is a child.""" 

187 ret = False 

188 if isinstance(dtype, tuple): 

189 for sub in dtype: 

190 ret = ret or cls._ischild(sub) 

191 elif isinstance(dtype, type) and issubclass(dtype, (Container, Data, DataRegion)): 

192 ret = True 

193 return ret 

194 

195 @staticmethod 

196 def _set_default_name(docval_args, default_name): 

197 """Set the default value for the name docval argument.""" 

198 if default_name is not None: 

199 for x in docval_args: 

200 if x['name'] == 'name': 

201 x['default'] = default_name 

202 

203 @classmethod 

204 def apply_generator_to_field(cls, field_spec, bases, type_map): 

205 """Return True to signal that this generator should return on all fields not yet processed.""" 

206 return True 

207 

208 @classmethod 

209 def process_field_spec(cls, classdict, docval_args, parent_cls, attr_name, not_inherited_fields, type_map, spec): 

210 """Add __fields__ to the classdict and update the docval args for the field spec with the given attribute name. 

211 :param classdict: The dict to update with __fields__ (or a different parent_cls._fieldsname). 

212 :param docval_args: The list of docval arguments. 

213 :param parent_cls: The parent class. 

214 :param attr_name: The attribute name of the field spec for the container class to generate. 

215 :param not_inherited_fields: Dictionary of fields not inherited from the parent class. 

216 :param type_map: The type map to use. 

217 :param spec: The spec for the container class to generate. 

218 """ 

219 field_spec = not_inherited_fields[attr_name] 

220 dtype = cls._get_type(field_spec, type_map) 

221 fields_conf = {'name': attr_name, 

222 'doc': field_spec['doc']} 

223 if cls._ischild(dtype) and issubclass(parent_cls, Container) and not isinstance(field_spec, LinkSpec): 

224 fields_conf['child'] = True 

225 # if getattr(field_spec, 'value', None) is not None: # TODO set the fixed value on the class? 

226 # fields_conf['settable'] = False 

227 classdict.setdefault(parent_cls._fieldsname, list()).append(fields_conf) 

228 

229 docval_arg = dict( 

230 name=attr_name, 

231 doc=field_spec.doc, 

232 type=cls._get_type(field_spec, type_map) 

233 ) 

234 shape = getattr(field_spec, 'shape', None) 

235 if shape is not None: 

236 docval_arg['shape'] = shape 

237 if cls._check_spec_optional(field_spec, spec): 

238 docval_arg['default'] = getattr(field_spec, 'default_value', None) 

239 cls._add_to_docval_args(docval_args, docval_arg) 

240 

241 @classmethod 

242 def _check_spec_optional(cls, field_spec, spec): 

243 """Returns True if the spec or any of its parents (up to the parent type spec) are optional.""" 

244 if not field_spec.required: 

245 return True 

246 if field_spec == spec: 

247 return False 

248 if field_spec.parent is not None: 

249 return cls._check_spec_optional(field_spec.parent, spec) 

250 

251 @classmethod 

252 def _add_to_docval_args(cls, docval_args, arg, err_if_present=False): 

253 """Add the docval arg to the list if not present. If present, overwrite it in place or raise an error.""" 

254 inserted = False 

255 for i, x in enumerate(docval_args): 

256 if x['name'] == arg['name']: 

257 if err_if_present: 257 ↛ 258line 257 didn't jump to line 258, because the condition on line 257 was never true

258 raise ValueError("Argument %s already exists in docval args." % arg["name"]) 

259 docval_args[i] = arg 

260 inserted = True 

261 if not inserted: 

262 docval_args.append(arg) 

263 

264 @classmethod 

265 def post_process(cls, classdict, bases, docval_args, spec): 

266 """Convert classdict['__fields__'] to tuple and update docval args for a fixed name and default name. 

267 :param classdict: The class dictionary to convert with '__fields__' key (or a different bases[0]._fieldsname) 

268 :param bases: The list of base classes. 

269 :param docval_args: The dict of docval arguments. 

270 :param spec: The spec for the container class to generate. 

271 """ 

272 # convert classdict['__fields__'] from list to tuple if present 

273 for b in bases: 

274 fields = classdict.get(b._fieldsname) 

275 if fields is not None and not isinstance(fields, tuple): 

276 classdict[b._fieldsname] = tuple(fields) 

277 

278 # if spec provides a fixed name for this type, remove the 'name' arg from docval_args so that values cannot 

279 # be passed for a name positional or keyword arg 

280 if spec.name is not None: 

281 for arg in list(docval_args): 

282 if arg['name'] == 'name': 

283 docval_args.remove(arg) 

284 

285 # set default name in docval args if provided 

286 cls._set_default_name(docval_args, spec.default_name) 

287 

288 @classmethod 

289 def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): 

290 # get docval arg names from superclass 

291 base = bases[0] 

292 parent_docval_args = set(arg['name'] for arg in get_docval(base.__init__)) 

293 new_args = list() 

294 for attr_name, field_spec in not_inherited_fields.items(): 

295 # store arguments for fields that are not in the superclass and not in the superclass __init__ docval 

296 # so that they are set after calling base.__init__ 

297 if attr_name not in parent_docval_args: 

298 new_args.append(attr_name) 

299 

300 @docval(*docval_args, allow_positional=AllowPositional.WARNING) 

301 def __init__(self, **kwargs): 

302 if name is not None: # force container name to be the fixed name in the spec 

303 kwargs.update(name=name) 

304 

305 # remove arguments from kwargs that correspond to fields that are new (not inherited) 

306 # set these arguments after calling base.__init__ 

307 new_kwargs = dict() 

308 for f in new_args: 

309 new_kwargs[f] = popargs(f, kwargs) if f in kwargs else None 

310 

311 # NOTE: the docval of some constructors do not include all of the fields. the constructor may set 

312 # some fields to fixed values. so only keep the kwargs that are used in the constructor docval 

313 kwargs_to_pass = {k: v for k, v in kwargs.items() if k in parent_docval_args} 

314 

315 base.__init__(self, **kwargs_to_pass) # special case: need to pass self to __init__ 

316 # TODO should super() be used above instead of base? 

317 

318 # set the fields that are new to this class (not inherited) 

319 for f, arg_val in new_kwargs.items(): 

320 setattr(self, f, arg_val) 

321 

322 classdict['__init__'] = __init__ 

323 

324 

325class MCIClassGenerator(CustomClassGenerator): 

326 

327 @classmethod 

328 def apply_generator_to_field(cls, field_spec, bases, type_map): 

329 """Return True if the field spec has quantity * or +, False otherwise.""" 

330 return getattr(field_spec, 'quantity', None) in (ZERO_OR_MANY, ONE_OR_MANY) 

331 

332 @classmethod 

333 def process_field_spec(cls, classdict, docval_args, parent_cls, attr_name, not_inherited_fields, type_map, spec): 

334 """Add __clsconf__ to the classdict and update the docval args for the field spec with the given attribute name. 

335 :param classdict: The dict to update with __clsconf__. 

336 :param docval_args: The list of docval arguments. 

337 :param parent_cls: The parent class. 

338 :param attr_name: The attribute name of the field spec for the container class to generate. 

339 :param not_inherited_fields: Dictionary of fields not inherited from the parent class. 

340 :param type_map: The type map to use. 

341 :param spec: The spec for the container class to generate. 

342 """ 

343 field_spec = not_inherited_fields[attr_name] 

344 field_clsconf = dict( 

345 attr=attr_name, 

346 type=cls._get_type(field_spec, type_map), 

347 add='add_{}'.format(attr_name), 

348 get='get_{}'.format(attr_name), 

349 create='create_{}'.format(attr_name) 

350 ) 

351 classdict.setdefault('__clsconf__', list()).append(field_clsconf) 

352 

353 # add a specialized docval arg for __init__ 

354 docval_arg = dict( 

355 name=attr_name, 

356 doc=field_spec.doc, 

357 type=(list, tuple, dict, cls._get_type(field_spec, type_map)) 

358 ) 

359 if cls._check_spec_optional(field_spec, spec): 

360 docval_arg['default'] = getattr(field_spec, 'default_value', None) 

361 cls._add_to_docval_args(docval_args, docval_arg) 

362 

363 @classmethod 

364 def post_process(cls, classdict, bases, docval_args, spec): 

365 """Add MultiContainerInterface to the list of base classes. 

366 :param classdict: The class dictionary. 

367 :param bases: The list of base classes. 

368 :param docval_args: The dict of docval arguments. 

369 :param spec: The spec for the container class to generate. 

370 """ 

371 if '__clsconf__' in classdict: 

372 # do not add MCI as a base if a base is already a subclass of MultiContainerInterface 

373 for b in bases: 

374 if issubclass(b, MultiContainerInterface): 

375 break 

376 else: 

377 if issubclass(MultiContainerInterface, bases[0]): 

378 # if bases[0] is Container or another superclass of MCI, then make sure MCI goes first 

379 # otherwise, MRO is ambiguous 

380 bases.insert(0, MultiContainerInterface) 

381 else: 

382 # bases[0] is not a subclass of MCI and not a superclass of MCI. place that class first 

383 # before MCI. that class __init__ should call super().__init__ which will call the 

384 # MCI init 

385 bases.insert(1, MultiContainerInterface) 

386 

387 @classmethod 

388 def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): 

389 if '__clsconf__' in classdict: 

390 previous_init = classdict['__init__'] 

391 

392 @docval(*docval_args, allow_positional=AllowPositional.WARNING) 

393 def __init__(self, **kwargs): 

394 # store the values passed to init for each MCI attribute so that they can be added 

395 # after calling __init__ 

396 new_kwargs = list() 

397 for field_clsconf in classdict['__clsconf__']: 

398 attr_name = field_clsconf['attr'] 

399 # do not store the value if it is None or not present 

400 if attr_name not in kwargs or kwargs[attr_name] is None: 

401 continue 

402 add_method_name = field_clsconf['add'] 

403 new_kwarg = dict( 

404 attr_name=attr_name, 

405 value=popargs(attr_name, kwargs), 

406 add_method_name=add_method_name 

407 ) 

408 new_kwargs.append(new_kwarg) 

409 

410 # pass an empty list to previous_init in case attr_name field is required 

411 # (one or many). we do not want previous_init to set the attribute directly. 

412 # instead, we will use the add_method after previous_init is finished. 

413 kwargs[attr_name] = list() 

414 

415 # call the parent class init without the MCI attribute 

416 previous_init(self, **kwargs) 

417 

418 # call the add method for each MCI attribute 

419 for new_kwarg in new_kwargs: 

420 add_method = getattr(self, new_kwarg['add_method_name']) 

421 add_method(new_kwarg['value']) 

422 

423 # override __init__ 

424 classdict['__init__'] = __init__