Coverage for src/configuraptor/core.py: 100%

184 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-26 13:54 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import math 

7import types 

8import typing 

9import warnings 

10from collections import ChainMap 

11from pathlib import Path 

12 

13from typeguard import TypeCheckError 

14from typeguard import check_type as _check_type 

15 

16from . import loaders 

17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey 

18from .helpers import camel_to_snake 

19from .postpone import Postponed 

20 

21# T is a reusable typevar 

22T = typing.TypeVar("T") 

23# t_typelike is anything that can be type hinted 

24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

25# t_data is anything that can be fed to _load_data 

26T_data = str | Path | dict[str, typing.Any] 

27# c = a config class instance, can be any (user-defined) class 

28C = typing.TypeVar("C") 

29# type c is a config class 

30Type_C = typing.Type[C] 

31 

32 

33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

34 """ 

35 If a key contains a dot, traverse the raw dict until the right key was found. 

36 

37 Example: 

38 key = some.nested.key 

39 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

40 -> {"with": "data"} 

41 """ 

42 parts = key.split(".") 

43 while parts: 

44 raw = raw[parts.pop(0)] 

45 

46 return raw 

47 

48 

49def _guess_key(clsname: str) -> str: 

50 """ 

51 If no key is manually defined for `load_into`, \ 

52 the class' name is converted to snake_case to use as the default key. 

53 """ 

54 return camel_to_snake(clsname) 

55 

56 

57def __load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

58 """ 

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

60 

61 E.g. class Tool will be mapped to key tool. 

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

63 """ 

64 if isinstance(data, str): 

65 data = Path(data) 

66 if isinstance(data, Path): 

67 with data.open("rb") as f: 

68 loader = loaders.get(data.suffix) 

69 data = loader(f) 

70 

71 if not data: 

72 return {} 

73 

74 if key is None: 

75 # try to guess key by grabbing the first one or using the class name 

76 if len(data) == 1: 

77 key = list(data.keys())[0] 

78 elif classname is not None: 

79 key = _guess_key(classname) 

80 

81 if key: 

82 data = _data_for_nested_key(key, data) 

83 

84 if not data: 

85 raise ValueError("No data found!") 

86 

87 if not isinstance(data, dict): 

88 raise ValueError("Data is not a dict!") 

89 

90 return data 

91 

92 

93def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

94 """ 

95 Wrapper around __load_data that retries with key="" if anything goes wrong. 

96 """ 

97 try: 

98 return __load_data(data, key, classname) 

99 except Exception as e: 

100 if key != "": 

101 return __load_data(data, "", classname) 

102 else: # pragma: no cover 

103 warnings.warn(f"Data could not be loaded: {e}", source=e) 

104 # key already was "", just return data! 

105 # (will probably not happen but fallback) 

106 return {} 

107 

108 

109def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

110 """ 

111 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

112 

113 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

114 """ 

115 try: 

116 _check_type(value, expected_type) 

117 return True 

118 except TypeCheckError: 

119 return False 

120 

121 

122def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]: 

123 """ 

124 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

125 

126 If an annotated key in missing from data, it will be filled with None for convenience. 

127 

128 TODO: python 3.11 exception groups to throw multiple errors at once! 

129 """ 

130 # custom object to use instead of None, since typing.Optional can be None! 

131 # cast to T to make mypy happy 

132 notfound = typing.cast(T, object()) 

133 postponed = Postponed() 

134 

135 final: dict[str, T | None] = {} 

136 for key, _type in annotations.items(): 

137 compare = data.get(key, notfound) 

138 if compare is notfound: # pragma: nocover 

139 warnings.warn( 

140 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

141 ) 

142 # skip! 

143 continue 

144 

145 if compare is postponed: 

146 # don't do anything with this item! 

147 continue 

148 

149 if not check_type(compare, _type): 

150 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

151 

152 final[key] = compare 

153 

154 return final 

155 

156 

157def convert_config(items: dict[str, T]) -> dict[str, T]: 

158 """ 

159 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

160 

161 1. removes any items where the value is None, since in that case the default should be used; 

162 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

163 """ 

164 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

165 

166 

167Type = typing.Type[typing.Any] 

168T_Type = typing.TypeVar("T_Type", bound=Type) 

169 

170 

171def is_builtin_type(_type: Type) -> bool: 

172 """ 

173 Returns whether _type is one of the builtin types. 

174 """ 

175 return _type.__module__ in ("__builtin__", "builtins") 

176 

177 

178# def is_builtin_class_instance(obj: typing.Any) -> bool: 

179# return is_builtin_type(obj.__class__) 

180 

181 

182def is_from_types_or_typing(_type: Type) -> bool: 

183 """ 

184 Returns whether _type is one of the stlib typing/types types. 

185 

186 e.g. types.UnionType or typing.Union 

187 """ 

188 return _type.__module__ in ("types", "typing") 

189 

190 

191def is_from_other_toml_supported_module(_type: Type) -> bool: 

192 """ 

193 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

194 so this returns whether _type is a type from these stdlib modules. 

195 """ 

196 return _type.__module__ in ("datetime", "math") 

197 

198 

199def is_parameterized(_type: Type) -> bool: 

200 """ 

201 Returns whether _type is a parameterized type. 

202 

203 Examples: 

204 list[str] -> True 

205 str -> False 

206 """ 

207 return typing.get_origin(_type) is not None 

208 

209 

210def is_custom_class(_type: Type) -> bool: 

211 """ 

212 Tries to guess if _type is a builtin or a custom (user-defined) class. 

213 

214 Other logic in this module depends on knowing that. 

215 """ 

216 return ( 

217 type(_type) is type 

218 and not is_builtin_type(_type) 

219 and not is_from_other_toml_supported_module(_type) 

220 and not is_from_types_or_typing(_type) 

221 ) 

222 

223 

224def instance_of_custom_class(var: typing.Any) -> bool: 

225 """ 

226 Calls `is_custom_class` on an instance of a (possibly custom) class. 

227 """ 

228 return is_custom_class(var.__class__) 

229 

230 

231def is_optional(_type: Type | typing.Any) -> bool: 

232 """ 

233 Tries to guess if _type could be optional. 

234 

235 Examples: 

236 None -> True 

237 NoneType -> True 

238 typing.Union[str, None] -> True 

239 str | None -> True 

240 list[str | None] -> False 

241 list[str] -> False 

242 """ 

243 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan): 

244 # e.g. list[str] 

245 # will crash issubclass to test it first here 

246 return False 

247 

248 return ( 

249 _type is None 

250 or types.NoneType in typing.get_args(_type) # union with Nonetype 

251 or issubclass(types.NoneType, _type) 

252 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

253 ) 

254 

255 

256def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

257 """ 

258 Get Field info for a dataclass cls. 

259 """ 

260 fields = getattr(cls, "__dataclass_fields__", {}) 

261 return fields.get(key) 

262 

263 

264def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]: 

265 """ 

266 For all annotations (recursively gathered from parents with `all_annotations`), \ 

267 try to resolve the tree of annotations. 

268 

269 Uses `load_into_recurse`, not itself directly. 

270 

271 Example: 

272 class First: 

273 key: str 

274 

275 class Second: 

276 other: First 

277 

278 # step 1 

279 cls = Second 

280 data = {"second": {"other": {"key": "anything"}}} 

281 annotations: {"other": First} 

282 

283 # step 1.5 

284 data = {"other": {"key": "anything"} 

285 annotations: {"other": First} 

286 

287 # step 2 

288 cls = First 

289 data = {"key": "anything"} 

290 annotations: {"key": str} 

291 

292 

293 TODO: python 3.11 exception groups to throw multiple errors at once! 

294 """ 

295 updated = {} 

296 

297 for _key, _type in annotations.items(): 

298 if _key in data: 

299 value: typing.Any = data[_key] # value can change so define it as any instead of T 

300 if is_parameterized(_type): 

301 origin = typing.get_origin(_type) 

302 arguments = typing.get_args(_type) 

303 if origin is list and arguments and is_custom_class(arguments[0]): 

304 subtype = arguments[0] 

305 value = [_load_into_recurse(subtype, subvalue) for subvalue in value] 

306 

307 elif origin is dict and arguments and is_custom_class(arguments[1]): 

308 # e.g. dict[str, Point] 

309 subkeytype, subvaluetype = arguments 

310 # subkey(type) is not a custom class, so don't try to convert it: 

311 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()} 

312 # elif origin is dict: 

313 # keep data the same 

314 elif origin is typing.Union and arguments: 

315 for arg in arguments: 

316 if is_custom_class(arg): 

317 value = _load_into_recurse(arg, value) 

318 else: 

319 # print(_type, arg, value) 

320 ... 

321 

322 # todo: other parameterized/unions/typing.Optional 

323 

324 elif is_custom_class(_type): 

325 # type must be C (custom class) at this point 

326 value = _load_into_recurse( 

327 # make mypy and pycharm happy by telling it _type is of type C... 

328 # actually just passing _type as first arg! 

329 typing.cast(Type_C[typing.Any], _type), 

330 value, 

331 ) 

332 

333 elif _key in cls.__dict__: 

334 # property has default, use that instead. 

335 value = cls.__dict__[_key] 

336 elif is_optional(_type): 

337 # type is optional and not found in __dict__ -> default is None 

338 value = None 

339 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

340 # could have a default factory 

341 # todo: do something with field.default? 

342 value = field.default_factory() 

343 else: 

344 raise ConfigErrorMissingKey(_key, cls, _type) 

345 

346 updated[_key] = value 

347 

348 return updated 

349 

350 

351def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

352 """ 

353 Returns a dictionary-like ChainMap that includes annotations for all \ 

354 attributes defined in cls or inherited from superclasses. 

355 """ 

356 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

357 

358 

359def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]: 

360 """ 

361 Wrapper around `_all_annotations` that filters away any keys in _except. 

362 

363 It also flattens the ChainMap to a regular dict. 

364 """ 

365 if _except is None: 

366 _except = set() 

367 

368 _all = _all_annotations(cls) 

369 return {k: v for k, v in _all.items() if k not in _except} 

370 

371 

372def check_and_convert_data( 

373 cls: typing.Type[C], 

374 data: dict[str, typing.Any], 

375 _except: typing.Iterable[str], 

376 strict: bool = True, 

377) -> dict[str, typing.Any]: 

378 """ 

379 Based on class annotations, this prepares the data for `load_into_recurse`. 

380 

381 1. convert config-keys to python compatible config_keys 

382 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

383 3. ensures the annotated types match the actual types after loading the config file. 

384 """ 

385 annotations = all_annotations(cls, _except=_except) 

386 

387 to_load = convert_config(data) 

388 to_load = load_recursive(cls, to_load, annotations) 

389 if strict: 

390 to_load = ensure_types(to_load, annotations) 

391 

392 return to_load 

393 

394 

395T_init_list = list[typing.Any] 

396T_init_dict = dict[str, typing.Any] 

397T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

398 

399 

400@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

401def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

402 """ 

403 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

404 """ 

405 if not init: 

406 return [], {} 

407 

408 args: T_init_list = [] 

409 kwargs: T_init_dict = {} 

410 match init: 

411 case (args, kwargs): 

412 return args, kwargs 

413 case [*args]: 

414 return args, {} 

415 case {**kwargs}: 

416 return [], kwargs 

417 case _: 

418 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

419 

420 

421def _load_into_recurse( 

422 cls: typing.Type[C], 

423 data: dict[str, typing.Any], 

424 init: T_init = None, 

425 strict: bool = True, 

426) -> C: 

427 """ 

428 Loads an instance of `cls` filled with `data`. 

429 

430 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

431 `init` can be used to optionally pass extra __init__ arguments. \ 

432 NOTE: This will overwrite a config key with the same name! 

433 """ 

434 init_args, init_kwargs = _split_init(init) 

435 

436 if dc.is_dataclass(cls): 

437 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict) 

438 if init: 

439 raise ValueError("Init is not allowed for dataclasses!") 

440 

441 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

442 inst = typing.cast(C, cls(**to_load)) 

443 else: 

444 inst = cls(*init_args, **init_kwargs) 

445 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict) 

446 inst.__dict__.update(**to_load) 

447 

448 return inst 

449 

450 

451def _load_into_instance( 

452 inst: C, 

453 cls: typing.Type[C], 

454 data: dict[str, typing.Any], 

455 init: T_init = None, 

456 strict: bool = True, 

457) -> C: 

458 """ 

459 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

460 and thus does not support init. 

461 

462 """ 

463 if init is not None: 

464 raise ValueError("Can not init an existing instance!") 

465 

466 existing_data = inst.__dict__ 

467 

468 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict) 

469 

470 inst.__dict__.update(**to_load) 

471 

472 return inst 

473 

474 

475def load_into_class( 

476 cls: typing.Type[C], 

477 data: T_data, 

478 /, 

479 key: str = None, 

480 init: T_init = None, 

481 strict: bool = True, 

482) -> C: 

483 """ 

484 Shortcut for _load_data + load_into_recurse. 

485 """ 

486 to_load = _load_data(data, key, cls.__name__) 

487 return _load_into_recurse(cls, to_load, init=init, strict=strict) 

488 

489 

490def load_into_instance( 

491 inst: C, 

492 data: T_data, 

493 /, 

494 key: str = None, 

495 init: T_init = None, 

496 strict: bool = True, 

497) -> C: 

498 """ 

499 Shortcut for _load_data + load_into_existing. 

500 """ 

501 cls = inst.__class__ 

502 to_load = _load_data(data, key, cls.__name__) 

503 return _load_into_instance(inst, cls, to_load, init=init, strict=strict) 

504 

505 

506def load_into( 

507 cls: typing.Type[C], 

508 data: T_data, 

509 /, 

510 key: str = None, 

511 init: T_init = None, 

512 strict: bool = True, 

513) -> C: 

514 """ 

515 Load your config into a class (instance). 

516 

517 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

518 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

519 

520 Args: 

521 cls: either a class or an existing instance of that class. 

522 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

523 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

524 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

525 strict: enable type checks or allow anything? 

526 

527 """ 

528 if not isinstance(cls, type): 

529 # would not be supported according to mypy, but you can still load_into(instance) 

530 return load_into_instance(cls, data, key=key, init=init, strict=strict) 

531 

532 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

533 # _cls = typing.cast(typing.Type[C], cls) 

534 return load_into_class(cls, data, key=key, init=init, strict=strict)