Coverage for src/configuraptor/core.py: 100%

212 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-03 15:36 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import math 

7import types 

8import typing 

9import warnings 

10from collections import ChainMap 

11from pathlib import Path 

12 

13from typeguard import TypeCheckError 

14from typeguard import check_type as _check_type 

15 

16from . import loaders 

17from .errors import ( 

18 ConfigErrorCouldNotConvert, 

19 ConfigErrorInvalidType, 

20 ConfigErrorMissingKey, 

21) 

22from .helpers import camel_to_snake 

23from .postpone import Postponed 

24 

25# T is a reusable typevar 

26T = typing.TypeVar("T") 

27# t_typelike is anything that can be type hinted 

28T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

29# t_data is anything that can be fed to _load_data 

30T_data = str | Path | dict[str, typing.Any] 

31# c = a config class instance, can be any (user-defined) class 

32C = typing.TypeVar("C") 

33# type c is a config class 

34Type_C = typing.Type[C] 

35 

36 

37def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

38 """ 

39 If a key contains a dot, traverse the raw dict until the right key was found. 

40 

41 Example: 

42 key = some.nested.key 

43 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

44 -> {"with": "data"} 

45 """ 

46 parts = key.split(".") 

47 while parts: 

48 raw = raw[parts.pop(0)] 

49 

50 return raw 

51 

52 

53def _guess_key(clsname: str) -> str: 

54 """ 

55 If no key is manually defined for `load_into`, \ 

56 the class' name is converted to snake_case to use as the default key. 

57 """ 

58 return camel_to_snake(clsname) 

59 

60 

61def __load_data( 

62 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False 

63) -> dict[str, typing.Any]: 

64 """ 

65 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

66 

67 E.g. class Tool will be mapped to key tool. 

68 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

69 """ 

70 if isinstance(data, str): 

71 data = Path(data) 

72 if isinstance(data, Path): 

73 with data.open("rb") as f: 

74 loader = loaders.get(data.suffix or data.name) 

75 data = loader(f, data.resolve()) 

76 

77 if not data: 

78 return {} 

79 

80 if key is None: 

81 # try to guess key by grabbing the first one or using the class name 

82 if len(data) == 1: 

83 key = list(data.keys())[0] 

84 elif classname is not None: 

85 key = _guess_key(classname) 

86 

87 if key: 

88 data = _data_for_nested_key(key, data) 

89 

90 if not data: 

91 raise ValueError("No data found!") 

92 

93 if not isinstance(data, dict): 

94 raise ValueError("Data is not a dict!") 

95 

96 if lower_keys: 

97 data = {k.lower(): v for k, v in data.items()} 

98 

99 return data 

100 

101 

102def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]: 

103 """ 

104 Wrapper around __load_data that retries with key="" if anything goes wrong. 

105 """ 

106 try: 

107 return __load_data(data, key, classname, lower_keys=lower_keys) 

108 except Exception as e: 

109 if key != "": 

110 return __load_data(data, "", classname, lower_keys=lower_keys) 

111 else: # pragma: no cover 

112 warnings.warn(f"Data could not be loaded: {e}", source=e) 

113 # key already was "", just return data! 

114 # (will probably not happen but fallback) 

115 return {} 

116 

117 

118def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

119 """ 

120 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

121 

122 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

123 """ 

124 try: 

125 _check_type(value, expected_type) 

126 return True 

127 except TypeCheckError: 

128 return False 

129 

130 

131F = typing.TypeVar("F") 

132 

133 

134def str_to_bool(value: str) -> bool: 

135 """ 

136 Used by convert_between, usually for .env loads. 

137 

138 Example: 

139 SOME_VALUE=TRUE -> True 

140 SOME_VALUE=1 -> True 

141 SOME_VALUE=Yes -> True 

142 

143 SOME_VALUE -> None 

144 SOME_VALUE=NOpe -> False 

145 

146 SOME_VALUE=Unrelated -> Error 

147 """ 

148 if not value: 

149 return False 

150 

151 first_letter = value[0].lower() 

152 # yes, true, 1 

153 if first_letter in {"y", "t", "1"}: 

154 return True 

155 elif first_letter in {"n", "f", "0"}: 

156 return False 

157 else: 

158 raise ValueError("Not booly.") 

159 

160 

161def str_to_none(value: str) -> typing.Optional[str]: 

162 """ 

163 Convert a string value of null/none to None, or keep the original string otherwise. 

164 """ 

165 if value.lower() in {"", "null", "none"}: 

166 return None 

167 else: 

168 return value 

169 

170 

171def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T: 

172 """ 

173 Convert a value between types. 

174 """ 

175 if from_type is str: 

176 if to_type is bool: 

177 return str_to_bool(from_value) # type: ignore 

178 elif to_type is None or to_type is types.NoneType: # noqa: E721 

179 return str_to_none(from_value) # type: ignore 

180 # default: just convert type: 

181 return to_type(from_value) # type: ignore 

182 

183 

184def ensure_types( 

185 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False 

186) -> dict[str, T | None]: 

187 """ 

188 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

189 

190 If an annotated key in missing from data, it will be filled with None for convenience. 

191 

192 TODO: python 3.11 exception groups to throw multiple errors at once! 

193 """ 

194 # custom object to use instead of None, since typing.Optional can be None! 

195 # cast to T to make mypy happy 

196 notfound = typing.cast(T, object()) 

197 postponed = Postponed() 

198 

199 final: dict[str, T | None] = {} 

200 for key, _type in annotations.items(): 

201 compare = data.get(key, notfound) 

202 if compare is notfound: # pragma: nocover 

203 warnings.warn( 

204 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

205 ) 

206 # skip! 

207 continue 

208 

209 if compare is postponed: 

210 # don't do anything with this item! 

211 continue 

212 

213 if not check_type(compare, _type): 

214 if convert_types: 

215 try: 

216 compare = convert_between(compare, type(compare), _type) 

217 except (TypeError, ValueError) as e: 

218 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e 

219 else: 

220 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

221 

222 final[key] = compare 

223 

224 return final 

225 

226 

227def convert_config(items: dict[str, T]) -> dict[str, T]: 

228 """ 

229 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

230 

231 1. removes any items where the value is None, since in that case the default should be used; 

232 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

233 """ 

234 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

235 

236 

237Type = typing.Type[typing.Any] 

238T_Type = typing.TypeVar("T_Type", bound=Type) 

239 

240 

241def is_builtin_type(_type: Type) -> bool: 

242 """ 

243 Returns whether _type is one of the builtin types. 

244 """ 

245 return _type.__module__ in ("__builtin__", "builtins") 

246 

247 

248# def is_builtin_class_instance(obj: typing.Any) -> bool: 

249# return is_builtin_type(obj.__class__) 

250 

251 

252def is_from_types_or_typing(_type: Type) -> bool: 

253 """ 

254 Returns whether _type is one of the stlib typing/types types. 

255 

256 e.g. types.UnionType or typing.Union 

257 """ 

258 return _type.__module__ in ("types", "typing") 

259 

260 

261def is_from_other_toml_supported_module(_type: Type) -> bool: 

262 """ 

263 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

264 so this returns whether _type is a type from these stdlib modules. 

265 """ 

266 return _type.__module__ in ("datetime", "math") 

267 

268 

269def is_parameterized(_type: Type) -> bool: 

270 """ 

271 Returns whether _type is a parameterized type. 

272 

273 Examples: 

274 list[str] -> True 

275 str -> False 

276 """ 

277 return typing.get_origin(_type) is not None 

278 

279 

280def is_custom_class(_type: Type) -> bool: 

281 """ 

282 Tries to guess if _type is a builtin or a custom (user-defined) class. 

283 

284 Other logic in this module depends on knowing that. 

285 """ 

286 return ( 

287 type(_type) is type 

288 and not is_builtin_type(_type) 

289 and not is_from_other_toml_supported_module(_type) 

290 and not is_from_types_or_typing(_type) 

291 ) 

292 

293 

294def instance_of_custom_class(var: typing.Any) -> bool: 

295 """ 

296 Calls `is_custom_class` on an instance of a (possibly custom) class. 

297 """ 

298 return is_custom_class(var.__class__) 

299 

300 

301def is_optional(_type: Type | typing.Any) -> bool: 

302 """ 

303 Tries to guess if _type could be optional. 

304 

305 Examples: 

306 None -> True 

307 NoneType -> True 

308 typing.Union[str, None] -> True 

309 str | None -> True 

310 list[str | None] -> False 

311 list[str] -> False 

312 """ 

313 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan): 

314 # e.g. list[str] 

315 # will crash issubclass to test it first here 

316 return False 

317 

318 return ( 

319 _type is None 

320 or types.NoneType in typing.get_args(_type) # union with Nonetype 

321 or issubclass(types.NoneType, _type) 

322 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

323 ) 

324 

325 

326def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

327 """ 

328 Get Field info for a dataclass cls. 

329 """ 

330 fields = getattr(cls, "__dataclass_fields__", {}) 

331 return fields.get(key) 

332 

333 

334def load_recursive( 

335 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False 

336) -> dict[str, T]: 

337 """ 

338 For all annotations (recursively gathered from parents with `all_annotations`), \ 

339 try to resolve the tree of annotations. 

340 

341 Uses `load_into_recurse`, not itself directly. 

342 

343 Example: 

344 class First: 

345 key: str 

346 

347 class Second: 

348 other: First 

349 

350 # step 1 

351 cls = Second 

352 data = {"second": {"other": {"key": "anything"}}} 

353 annotations: {"other": First} 

354 

355 # step 1.5 

356 data = {"other": {"key": "anything"} 

357 annotations: {"other": First} 

358 

359 # step 2 

360 cls = First 

361 data = {"key": "anything"} 

362 annotations: {"key": str} 

363 

364 

365 TODO: python 3.11 exception groups to throw multiple errors at once! 

366 """ 

367 updated = {} 

368 

369 for _key, _type in annotations.items(): 

370 if _key in data: 

371 value: typing.Any = data[_key] # value can change so define it as any instead of T 

372 if is_parameterized(_type): 

373 origin = typing.get_origin(_type) 

374 arguments = typing.get_args(_type) 

375 if origin is list and arguments and is_custom_class(arguments[0]): 

376 subtype = arguments[0] 

377 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value] 

378 

379 elif origin is dict and arguments and is_custom_class(arguments[1]): 

380 # e.g. dict[str, Point] 

381 subkeytype, subvaluetype = arguments 

382 # subkey(type) is not a custom class, so don't try to convert it: 

383 value = { 

384 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types) 

385 for subkey, subvalue in value.items() 

386 } 

387 # elif origin is dict: 

388 # keep data the same 

389 elif origin is typing.Union and arguments: 

390 for arg in arguments: 

391 if is_custom_class(arg): 

392 value = _load_into_recurse(arg, value, convert_types=convert_types) 

393 else: 

394 # print(_type, arg, value) 

395 ... 

396 

397 # todo: other parameterized/unions/typing.Optional 

398 

399 elif is_custom_class(_type): 

400 # type must be C (custom class) at this point 

401 value = _load_into_recurse( 

402 # make mypy and pycharm happy by telling it _type is of type C... 

403 # actually just passing _type as first arg! 

404 typing.cast(Type_C[typing.Any], _type), 

405 value, 

406 convert_types=convert_types, 

407 ) 

408 

409 elif _key in cls.__dict__: 

410 # property has default, use that instead. 

411 value = cls.__dict__[_key] 

412 elif is_optional(_type): 

413 # type is optional and not found in __dict__ -> default is None 

414 value = None 

415 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

416 # could have a default factory 

417 # todo: do something with field.default? 

418 value = field.default_factory() 

419 else: 

420 raise ConfigErrorMissingKey(_key, cls, _type) 

421 

422 updated[_key] = value 

423 

424 return updated 

425 

426 

427def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

428 """ 

429 Returns a dictionary-like ChainMap that includes annotations for all \ 

430 attributes defined in cls or inherited from superclasses. 

431 """ 

432 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

433 

434 

435def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, type[object]]: 

436 """ 

437 Wrapper around `_all_annotations` that filters away any keys in _except. 

438 

439 It also flattens the ChainMap to a regular dict. 

440 """ 

441 if _except is None: 

442 _except = set() 

443 

444 _all = _all_annotations(cls) 

445 return {k: v for k, v in _all.items() if k not in _except} 

446 

447 

448def check_and_convert_data( 

449 cls: typing.Type[C], 

450 data: dict[str, typing.Any], 

451 _except: typing.Iterable[str], 

452 strict: bool = True, 

453 convert_types: bool = False, 

454) -> dict[str, typing.Any]: 

455 """ 

456 Based on class annotations, this prepares the data for `load_into_recurse`. 

457 

458 1. convert config-keys to python compatible config_keys 

459 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

460 3. ensures the annotated types match the actual types after loading the config file. 

461 """ 

462 annotations = all_annotations(cls, _except=_except) 

463 

464 to_load = convert_config(data) 

465 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types) 

466 if strict: 

467 to_load = ensure_types(to_load, annotations, convert_types=convert_types) 

468 

469 return to_load 

470 

471 

472T_init_list = list[typing.Any] 

473T_init_dict = dict[str, typing.Any] 

474T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

475 

476 

477@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

478def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

479 """ 

480 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

481 """ 

482 if not init: 

483 return [], {} 

484 

485 args: T_init_list = [] 

486 kwargs: T_init_dict = {} 

487 match init: 

488 case (args, kwargs): 

489 return args, kwargs 

490 case [*args]: 

491 return args, {} 

492 case {**kwargs}: 

493 return [], kwargs 

494 case _: 

495 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

496 

497 

498def _load_into_recurse( 

499 cls: typing.Type[C], 

500 data: dict[str, typing.Any], 

501 init: T_init = None, 

502 strict: bool = True, 

503 convert_types: bool = False, 

504) -> C: 

505 """ 

506 Loads an instance of `cls` filled with `data`. 

507 

508 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

509 `init` can be used to optionally pass extra __init__ arguments. \ 

510 NOTE: This will overwrite a config key with the same name! 

511 """ 

512 init_args, init_kwargs = _split_init(init) 

513 

514 if dc.is_dataclass(cls): 

515 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types) 

516 if init: 

517 raise ValueError("Init is not allowed for dataclasses!") 

518 

519 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

520 inst = typing.cast(C, cls(**to_load)) 

521 else: 

522 inst = cls(*init_args, **init_kwargs) 

523 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types) 

524 inst.__dict__.update(**to_load) 

525 

526 return inst 

527 

528 

529def _load_into_instance( 

530 inst: C, 

531 cls: typing.Type[C], 

532 data: dict[str, typing.Any], 

533 init: T_init = None, 

534 strict: bool = True, 

535 convert_types: bool = False, 

536) -> C: 

537 """ 

538 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

539 and thus does not support init. 

540 

541 """ 

542 if init is not None: 

543 raise ValueError("Can not init an existing instance!") 

544 

545 existing_data = inst.__dict__ 

546 

547 to_load = check_and_convert_data( 

548 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types 

549 ) 

550 

551 inst.__dict__.update(**to_load) 

552 

553 return inst 

554 

555 

556def load_into_class( 

557 cls: typing.Type[C], 

558 data: T_data, 

559 /, 

560 key: str = None, 

561 init: T_init = None, 

562 strict: bool = True, 

563 lower_keys: bool = False, 

564 convert_types: bool = False, 

565) -> C: 

566 """ 

567 Shortcut for _load_data + load_into_recurse. 

568 """ 

569 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys) 

570 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types) 

571 

572 

573def load_into_instance( 

574 inst: C, 

575 data: T_data, 

576 /, 

577 key: str = None, 

578 init: T_init = None, 

579 strict: bool = True, 

580 lower_keys: bool = False, 

581 convert_types: bool = False, 

582) -> C: 

583 """ 

584 Shortcut for _load_data + load_into_existing. 

585 """ 

586 cls = inst.__class__ 

587 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys) 

588 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types) 

589 

590 

591def load_into( 

592 cls: typing.Type[C], 

593 data: T_data, 

594 /, 

595 key: str = None, 

596 init: T_init = None, 

597 strict: bool = True, 

598 lower_keys: bool = False, 

599 convert_types: bool = False, 

600) -> C: 

601 """ 

602 Load your config into a class (instance). 

603 

604 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

605 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

606 

607 Args: 

608 cls: either a class or an existing instance of that class. 

609 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

610 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

611 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

612 strict: enable type checks or allow anything? 

613 lower_keys: should the config keys be lowercased? (for .env) 

614 convert_types: should the types be converted to the annotated type if not yet matching? (for .env) 

615 

616 """ 

617 if not isinstance(cls, type): 

618 # would not be supported according to mypy, but you can still load_into(instance) 

619 return load_into_instance( 

620 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

621 ) 

622 

623 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

624 # _cls = typing.cast(typing.Type[C], cls) 

625 return load_into_class( 

626 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

627 )