Coverage for src/configuraptor/core.py: 100%

153 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-15 14:30 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import types 

7import typing 

8import warnings 

9from collections import ChainMap 

10from pathlib import Path 

11 

12from typeguard import TypeCheckError 

13from typeguard import check_type as _check_type 

14 

15from . import loaders 

16from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey 

17from .helpers import camel_to_snake 

18from .postpone import Postponed 

19 

20# T is a reusable typevar 

21T = typing.TypeVar("T") 

22# t_typelike is anything that can be type hinted 

23T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

24# t_data is anything that can be fed to _load_data 

25T_data = str | Path | dict[str, typing.Any] 

26# c = a config class instance, can be any (user-defined) class 

27C = typing.TypeVar("C") 

28# type c is a config class 

29Type_C = typing.Type[C] 

30 

31 

32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

33 """ 

34 If a key contains a dot, traverse the raw dict until the right key was found. 

35 

36 Example: 

37 key = some.nested.key 

38 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

39 -> {"with": "data"} 

40 """ 

41 parts = key.split(".") 

42 while parts: 

43 raw = raw[parts.pop(0)] 

44 

45 return raw 

46 

47 

48def _guess_key(clsname: str) -> str: 

49 """ 

50 If no key is manually defined for `load_into`, \ 

51 the class' name is converted to snake_case to use as the default key. 

52 """ 

53 return camel_to_snake(clsname) 

54 

55 

56def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

57 """ 

58 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

59 

60 E.g. class Tool will be mapped to key tool. 

61 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

62 """ 

63 if isinstance(data, str): 

64 data = Path(data) 

65 if isinstance(data, Path): 

66 with data.open("rb") as f: 

67 loader = loaders.get(data.suffix) 

68 data = loader(f) 

69 

70 if not data: 

71 return {} 

72 

73 if key is None: 

74 # try to guess key by grabbing the first one or using the class name 

75 if len(data) == 1: 

76 key = list(data.keys())[0] 

77 elif classname is not None: 

78 key = _guess_key(classname) 

79 

80 if key: 

81 return _data_for_nested_key(key, data) 

82 else: 

83 # no key found, just return all data 

84 return data 

85 

86 

87def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

88 """ 

89 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

90 

91 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

92 """ 

93 try: 

94 _check_type(value, expected_type) 

95 return True 

96 except TypeCheckError: 

97 return False 

98 

99 

100def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]: 

101 """ 

102 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

103 

104 If an annotated key in missing from data, it will be filled with None for convenience. 

105 """ 

106 # custom object to use instead of None, since typing.Optional can be None! 

107 # cast to T to make mypy happy 

108 notfound = typing.cast(T, object()) 

109 postponed = Postponed() 

110 

111 final: dict[str, T | None] = {} 

112 for key, _type in annotations.items(): 

113 compare = data.get(key, notfound) 

114 if compare is notfound: # pragma: nocover 

115 warnings.warn( 

116 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

117 ) 

118 # skip! 

119 continue 

120 

121 if compare is postponed: 

122 # don't do anything with this item! 

123 continue 

124 

125 if not check_type(compare, _type): 

126 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

127 

128 final[key] = compare 

129 return final 

130 

131 

132def convert_config(items: dict[str, T]) -> dict[str, T]: 

133 """ 

134 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

135 

136 1. removes any items where the value is None, since in that case the default should be used; 

137 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

138 """ 

139 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

140 

141 

142Type = typing.Type[typing.Any] 

143T_Type = typing.TypeVar("T_Type", bound=Type) 

144 

145 

146def is_builtin_type(_type: Type) -> bool: 

147 """ 

148 Returns whether _type is one of the builtin types. 

149 """ 

150 return _type.__module__ in ("__builtin__", "builtins") 

151 

152 

153# def is_builtin_class_instance(obj: typing.Any) -> bool: 

154# return is_builtin_type(obj.__class__) 

155 

156 

157def is_from_types_or_typing(_type: Type) -> bool: 

158 """ 

159 Returns whether _type is one of the stlib typing/types types. 

160 

161 e.g. types.UnionType or typing.Union 

162 """ 

163 return _type.__module__ in ("types", "typing") 

164 

165 

166def is_from_other_toml_supported_module(_type: Type) -> bool: 

167 """ 

168 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

169 so this returns whether _type is a type from these stdlib modules. 

170 """ 

171 return _type.__module__ in ("datetime", "math") 

172 

173 

174def is_parameterized(_type: Type) -> bool: 

175 """ 

176 Returns whether _type is a parameterized type. 

177 

178 Examples: 

179 list[str] -> True 

180 str -> False 

181 """ 

182 return typing.get_origin(_type) is not None 

183 

184 

185def is_custom_class(_type: Type) -> bool: 

186 """ 

187 Tries to guess if _type is a builtin or a custom (user-defined) class. 

188 

189 Other logic in this module depends on knowing that. 

190 """ 

191 return ( 

192 type(_type) is type 

193 and not is_builtin_type(_type) 

194 and not is_from_other_toml_supported_module(_type) 

195 and not is_from_types_or_typing(_type) 

196 ) 

197 

198 

199def is_optional(_type: Type | None) -> bool: 

200 """ 

201 Tries to guess if _type could be optional. 

202 

203 Examples: 

204 None -> True 

205 NoneType -> True 

206 typing.Union[str, None] -> True 

207 str | None -> True 

208 list[str | None] -> False 

209 list[str] -> False 

210 """ 

211 if _type and is_parameterized(_type) and typing.get_origin(_type) in (dict, list): 

212 # e.g. list[str] 

213 # will crash issubclass to test it first here 

214 return False 

215 

216 return ( 

217 _type is None 

218 or issubclass(types.NoneType, _type) 

219 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

220 or type(None) in typing.get_args(_type) # union with Nonetype 

221 ) 

222 

223 

224def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

225 """ 

226 Get Field info for a dataclass cls. 

227 """ 

228 fields = getattr(cls, "__dataclass_fields__", {}) 

229 return fields.get(key) 

230 

231 

232def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]: 

233 """ 

234 For all annotations (recursively gathered from parents with `all_annotations`), \ 

235 try to resolve the tree of annotations. 

236 

237 Uses `load_into_recurse`, not itself directly. 

238 

239 Example: 

240 class First: 

241 key: str 

242 

243 class Second: 

244 other: First 

245 

246 # step 1 

247 cls = Second 

248 data = {"second": {"other": {"key": "anything"}}} 

249 annotations: {"other": First} 

250 

251 # step 1.5 

252 data = {"other": {"key": "anything"} 

253 annotations: {"other": First} 

254 

255 # step 2 

256 cls = First 

257 data = {"key": "anything"} 

258 annotations: {"key": str} 

259 

260 """ 

261 updated = {} 

262 for _key, _type in annotations.items(): 

263 if _key in data: 

264 value: typing.Any = data[_key] # value can change so define it as any instead of T 

265 if is_parameterized(_type): 

266 origin = typing.get_origin(_type) 

267 arguments = typing.get_args(_type) 

268 if origin is list and arguments and is_custom_class(arguments[0]): 

269 subtype = arguments[0] 

270 value = [_load_into_recurse(subtype, subvalue) for subvalue in value] 

271 

272 elif origin is dict and arguments and is_custom_class(arguments[1]): 

273 # e.g. dict[str, Point] 

274 subkeytype, subvaluetype = arguments 

275 # subkey(type) is not a custom class, so don't try to convert it: 

276 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()} 

277 # elif origin is dict: 

278 # keep data the same 

279 elif origin is typing.Union and arguments: 

280 for arg in arguments: 

281 if is_custom_class(arg): 

282 value = _load_into_recurse(arg, value) 

283 else: 

284 # print(_type, arg, value) 

285 ... 

286 

287 # todo: other parameterized/unions/typing.Optional 

288 

289 elif is_custom_class(_type): 

290 # type must be C (custom class) at this point 

291 value = _load_into_recurse( 

292 # make mypy and pycharm happy by telling it _type is of type C... 

293 # actually just passing _type as first arg! 

294 typing.cast(Type_C[typing.Any], _type), 

295 value, 

296 ) 

297 

298 elif _key in cls.__dict__: 

299 # property has default, use that instead. 

300 value = cls.__dict__[_key] 

301 elif is_optional(_type): 

302 # type is optional and not found in __dict__ -> default is None 

303 value = None 

304 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

305 # could have a default factory 

306 # todo: do something with field.default? 

307 value = field.default_factory() 

308 else: 

309 # todo: exception group? 

310 raise ConfigErrorMissingKey(_key, cls, _type) 

311 

312 updated[_key] = value 

313 

314 return updated 

315 

316 

317def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

318 """ 

319 Returns a dictionary-like ChainMap that includes annotations for all \ 

320 attributes defined in cls or inherited from superclasses. 

321 """ 

322 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

323 

324 

325def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]: 

326 """ 

327 Wrapper around `_all_annotations` that filters away any keys in _except. 

328 

329 It also flattens the ChainMap to a regular dict. 

330 """ 

331 if _except is None: 

332 _except = set() 

333 

334 _all = _all_annotations(cls) 

335 return {k: v for k, v in _all.items() if k not in _except} 

336 

337 

338def check_and_convert_data( 

339 cls: typing.Type[C], 

340 data: dict[str, typing.Any], 

341 _except: typing.Iterable[str], 

342 strict: bool = True, 

343) -> dict[str, typing.Any]: 

344 """ 

345 Based on class annotations, this prepares the data for `load_into_recurse`. 

346 

347 1. convert config-keys to python compatible config_keys 

348 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

349 3. ensures the annotated types match the actual types after loading the config file. 

350 """ 

351 annotations = all_annotations(cls, _except=_except) 

352 

353 to_load = convert_config(data) 

354 to_load = load_recursive(cls, to_load, annotations) 

355 if strict: 

356 to_load = ensure_types(to_load, annotations) 

357 

358 return to_load 

359 

360 

361def _load_into_recurse( 

362 cls: typing.Type[C], 

363 data: dict[str, typing.Any], 

364 init: dict[str, typing.Any] = None, 

365 strict: bool = True, 

366) -> C: 

367 """ 

368 Loads an instance of `cls` filled with `data`. 

369 

370 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

371 `init` can be used to optionally pass extra __init__ arguments. \ 

372 NOTE: This will overwrite a config key with the same name! 

373 """ 

374 if init is None: 

375 init = {} 

376 

377 # fixme: cls.__init__ can set other keys than the name is in kwargs!! 

378 

379 if dc.is_dataclass(cls): 

380 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict) 

381 to_load |= init # add extra init variables (should not happen for a dataclass but whatev) 

382 

383 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

384 inst = typing.cast(C, cls(**to_load)) 

385 else: 

386 inst = cls(**init) 

387 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict) 

388 inst.__dict__.update(**to_load) 

389 

390 return inst 

391 

392 

393def _load_into_instance( 

394 inst: C, 

395 cls: typing.Type[C], 

396 data: dict[str, typing.Any], 

397 init: dict[str, typing.Any] = None, 

398 strict: bool = True, 

399) -> C: 

400 """ 

401 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

402 and thus does not support init. 

403 

404 """ 

405 if init is not None: 

406 raise ValueError("Can not init an existing instance!") 

407 

408 existing_data = inst.__dict__ 

409 

410 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict) 

411 

412 inst.__dict__.update(**to_load) 

413 

414 return inst 

415 

416 

417def load_into_class( 

418 cls: typing.Type[C], 

419 data: T_data, 

420 /, 

421 key: str = None, 

422 init: dict[str, typing.Any] = None, 

423 strict: bool = True, 

424) -> C: 

425 """ 

426 Shortcut for _load_data + load_into_recurse. 

427 """ 

428 to_load = _load_data(data, key, cls.__name__) 

429 return _load_into_recurse(cls, to_load, init=init, strict=strict) 

430 

431 

432def load_into_instance( 

433 inst: C, 

434 data: T_data, 

435 /, 

436 key: str = None, 

437 init: dict[str, typing.Any] = None, 

438 strict: bool = True, 

439) -> C: 

440 """ 

441 Shortcut for _load_data + load_into_existing. 

442 """ 

443 cls = inst.__class__ 

444 to_load = _load_data(data, key, cls.__name__) 

445 return _load_into_instance(inst, cls, to_load, init=init, strict=strict) 

446 

447 

448def load_into( 

449 cls: typing.Type[C], 

450 data: T_data, 

451 /, 

452 key: str = None, 

453 init: dict[str, typing.Any] = None, 

454 strict: bool = True, 

455) -> C: 

456 """ 

457 Load your config into a class (instance). 

458 

459 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

460 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

461 

462 Args: 

463 cls: either a class or an existing instance of that class. 

464 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

465 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

466 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

467 strict: enable type checks or allow anything? 

468 

469 """ 

470 if not isinstance(cls, type): 

471 # would not be supported according to mypy, but you can still load_into(instance) 

472 return load_into_instance(cls, data, key=key, init=init, strict=strict) 

473 

474 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

475 # _cls = typing.cast(typing.Type[C], cls) 

476 return load_into_class(cls, data, key=key, init=init, strict=strict)