Coverage for src/configuraptor/core.py: 100%

202 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 10:22 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import math 

7import types 

8import typing 

9import warnings 

10from collections import ChainMap 

11from pathlib import Path 

12 

13from typeguard import TypeCheckError 

14from typeguard import check_type as _check_type 

15 

16from . import loaders 

17from .errors import ( 

18 ConfigErrorCouldNotConvert, 

19 ConfigErrorInvalidType, 

20 ConfigErrorMissingKey, 

21) 

22from .helpers import camel_to_snake, find_pyproject_toml 

23from .postpone import Postponed 

24from .type_converters import CONVERTERS 

25 

26# T is a reusable typevar 

27T = typing.TypeVar("T") 

28# t_typelike is anything that can be type hinted 

29T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

30# t_data is anything that can be fed to _load_data 

31T_data = str | Path | dict[str, typing.Any] | None 

32# c = a config class instance, can be any (user-defined) class 

33C = typing.TypeVar("C") 

34# type c is a config class 

35Type_C = typing.Type[C] 

36 

37 

38def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

39 """ 

40 If a key contains a dot, traverse the raw dict until the right key was found. 

41 

42 Example: 

43 key = some.nested.key 

44 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

45 -> {"with": "data"} 

46 """ 

47 parts = key.split(".") 

48 while parts: 

49 key = parts.pop(0) 

50 if key not in raw: 

51 return {} 

52 

53 raw = raw[key] 

54 

55 return raw 

56 

57 

58def _guess_key(clsname: str) -> str: 

59 """ 

60 If no key is manually defined for `load_into`, \ 

61 the class' name is converted to snake_case to use as the default key. 

62 """ 

63 return camel_to_snake(clsname) 

64 

65 

66def __load_data( 

67 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False 

68) -> dict[str, typing.Any]: 

69 """ 

70 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

71 

72 E.g. class Tool will be mapped to key tool. 

73 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

74 """ 

75 if isinstance(data, str): 

76 data = Path(data) 

77 if isinstance(data, Path): 

78 with data.open("rb") as f: 

79 loader = loaders.get(data.suffix or data.name) 

80 data = loader(f, data.resolve()) 

81 if not data: 

82 return {} 

83 

84 if key is None: 

85 # try to guess key by grabbing the first one or using the class name 

86 if len(data) == 1: 

87 key = list(data.keys())[0] 

88 elif classname is not None: 

89 key = _guess_key(classname) 

90 

91 if key: 

92 data = _data_for_nested_key(key, data) 

93 

94 if not data: 

95 raise ValueError("No data found!") 

96 

97 if not isinstance(data, dict): 

98 raise ValueError("Data is not a dict!") 

99 

100 if lower_keys: 

101 data = {k.lower(): v for k, v in data.items()} 

102 

103 return data 

104 

105 

106def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]: 

107 """ 

108 Wrapper around __load_data that retries with key="" if anything goes wrong. 

109 """ 

110 if data is None: 

111 # try to load pyproject.toml 

112 data = find_pyproject_toml() 

113 

114 try: 

115 return __load_data(data, key, classname, lower_keys=lower_keys) 

116 except Exception as e: 

117 if key != "": 

118 return __load_data(data, "", classname, lower_keys=lower_keys) 

119 else: # pragma: no cover 

120 warnings.warn(f"Data could not be loaded: {e}", source=e) 

121 # key already was "", just return data! 

122 # (will probably not happen but fallback) 

123 return {} 

124 

125 

126def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

127 """ 

128 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

129 

130 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

131 """ 

132 try: 

133 _check_type(value, expected_type) 

134 return True 

135 except TypeCheckError: 

136 return False 

137 

138 

139F = typing.TypeVar("F") 

140 

141 

142def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T: 

143 """ 

144 Convert a value between types. 

145 """ 

146 if converter := CONVERTERS.get((from_type, to_type)): 

147 return typing.cast(T, converter(from_value)) 

148 

149 # default: just convert type: 

150 return to_type(from_value) # type: ignore 

151 

152 

153def ensure_types( 

154 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False 

155) -> dict[str, T | None]: 

156 """ 

157 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

158 

159 If an annotated key in missing from data, it will be filled with None for convenience. 

160 

161 TODO: python 3.11 exception groups to throw multiple errors at once! 

162 """ 

163 # custom object to use instead of None, since typing.Optional can be None! 

164 # cast to T to make mypy happy 

165 notfound = typing.cast(T, object()) 

166 postponed = Postponed() 

167 

168 final: dict[str, T | None] = {} 

169 for key, _type in annotations.items(): 

170 compare = data.get(key, notfound) 

171 if compare is notfound: # pragma: nocover 

172 warnings.warn( 

173 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

174 ) 

175 # skip! 

176 continue 

177 

178 if compare is postponed: 

179 # don't do anything with this item! 

180 continue 

181 

182 if not check_type(compare, _type): 

183 if convert_types: 

184 try: 

185 compare = convert_between(compare, type(compare), _type) 

186 except (TypeError, ValueError) as e: 

187 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e 

188 else: 

189 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

190 

191 final[key] = compare 

192 

193 return final 

194 

195 

196def convert_config(items: dict[str, T]) -> dict[str, T]: 

197 """ 

198 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

199 

200 1. removes any items where the value is None, since in that case the default should be used; 

201 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

202 """ 

203 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

204 

205 

206Type = typing.Type[typing.Any] 

207T_Type = typing.TypeVar("T_Type", bound=Type) 

208 

209 

210def is_builtin_type(_type: Type) -> bool: 

211 """ 

212 Returns whether _type is one of the builtin types. 

213 """ 

214 return _type.__module__ in ("__builtin__", "builtins") 

215 

216 

217# def is_builtin_class_instance(obj: typing.Any) -> bool: 

218# return is_builtin_type(obj.__class__) 

219 

220 

221def is_from_types_or_typing(_type: Type) -> bool: 

222 """ 

223 Returns whether _type is one of the stlib typing/types types. 

224 

225 e.g. types.UnionType or typing.Union 

226 """ 

227 return _type.__module__ in ("types", "typing") 

228 

229 

230def is_from_other_toml_supported_module(_type: Type) -> bool: 

231 """ 

232 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

233 so this returns whether _type is a type from these stdlib modules. 

234 """ 

235 return _type.__module__ in ("datetime", "math") 

236 

237 

238def is_parameterized(_type: Type) -> bool: 

239 """ 

240 Returns whether _type is a parameterized type. 

241 

242 Examples: 

243 list[str] -> True 

244 str -> False 

245 """ 

246 return typing.get_origin(_type) is not None 

247 

248 

249def is_custom_class(_type: Type) -> bool: 

250 """ 

251 Tries to guess if _type is a builtin or a custom (user-defined) class. 

252 

253 Other logic in this module depends on knowing that. 

254 """ 

255 return ( 

256 type(_type) is type 

257 and not is_builtin_type(_type) 

258 and not is_from_other_toml_supported_module(_type) 

259 and not is_from_types_or_typing(_type) 

260 ) 

261 

262 

263def instance_of_custom_class(var: typing.Any) -> bool: 

264 """ 

265 Calls `is_custom_class` on an instance of a (possibly custom) class. 

266 """ 

267 return is_custom_class(var.__class__) 

268 

269 

270def is_optional(_type: Type | typing.Any) -> bool: 

271 """ 

272 Tries to guess if _type could be optional. 

273 

274 Examples: 

275 None -> True 

276 NoneType -> True 

277 typing.Union[str, None] -> True 

278 str | None -> True 

279 list[str | None] -> False 

280 list[str] -> False 

281 """ 

282 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan): 

283 # e.g. list[str] 

284 # will crash issubclass to test it first here 

285 return False 

286 

287 return ( 

288 _type is None 

289 or types.NoneType in typing.get_args(_type) # union with Nonetype 

290 or issubclass(types.NoneType, _type) 

291 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

292 ) 

293 

294 

295def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

296 """ 

297 Get Field info for a dataclass cls. 

298 """ 

299 fields = getattr(cls, "__dataclass_fields__", {}) 

300 return fields.get(key) 

301 

302 

303def load_recursive( 

304 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False 

305) -> dict[str, T]: 

306 """ 

307 For all annotations (recursively gathered from parents with `all_annotations`), \ 

308 try to resolve the tree of annotations. 

309 

310 Uses `load_into_recurse`, not itself directly. 

311 

312 Example: 

313 class First: 

314 key: str 

315 

316 class Second: 

317 other: First 

318 

319 # step 1 

320 cls = Second 

321 data = {"second": {"other": {"key": "anything"}}} 

322 annotations: {"other": First} 

323 

324 # step 1.5 

325 data = {"other": {"key": "anything"} 

326 annotations: {"other": First} 

327 

328 # step 2 

329 cls = First 

330 data = {"key": "anything"} 

331 annotations: {"key": str} 

332 

333 

334 TODO: python 3.11 exception groups to throw multiple errors at once! 

335 """ 

336 updated = {} 

337 

338 for _key, _type in annotations.items(): 

339 if _key in data: 

340 value: typing.Any = data[_key] # value can change so define it as any instead of T 

341 if is_parameterized(_type): 

342 origin = typing.get_origin(_type) 

343 arguments = typing.get_args(_type) 

344 if origin is list and arguments and is_custom_class(arguments[0]): 

345 subtype = arguments[0] 

346 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value] 

347 

348 elif origin is dict and arguments and is_custom_class(arguments[1]): 

349 # e.g. dict[str, Point] 

350 subkeytype, subvaluetype = arguments 

351 # subkey(type) is not a custom class, so don't try to convert it: 

352 value = { 

353 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types) 

354 for subkey, subvalue in value.items() 

355 } 

356 # elif origin is dict: 

357 # keep data the same 

358 elif origin is typing.Union and arguments: 

359 for arg in arguments: 

360 if is_custom_class(arg): 

361 value = _load_into_recurse(arg, value, convert_types=convert_types) 

362 else: 

363 # print(_type, arg, value) 

364 ... 

365 

366 # todo: other parameterized/unions/typing.Optional 

367 

368 elif is_custom_class(_type): 

369 # type must be C (custom class) at this point 

370 value = _load_into_recurse( 

371 # make mypy and pycharm happy by telling it _type is of type C... 

372 # actually just passing _type as first arg! 

373 typing.cast(Type_C[typing.Any], _type), 

374 value, 

375 convert_types=convert_types, 

376 ) 

377 

378 elif _key in cls.__dict__: 

379 # property has default, use that instead. 

380 value = cls.__dict__[_key] 

381 elif is_optional(_type): 

382 # type is optional and not found in __dict__ -> default is None 

383 value = None 

384 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

385 # could have a default factory 

386 # todo: do something with field.default? 

387 value = field.default_factory() 

388 else: 

389 raise ConfigErrorMissingKey(_key, cls, _type) 

390 

391 updated[_key] = value 

392 

393 return updated 

394 

395 

396def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

397 """ 

398 Returns a dictionary-like ChainMap that includes annotations for all \ 

399 attributes defined in cls or inherited from superclasses. 

400 """ 

401 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

402 

403 

404def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, type[object]]: 

405 """ 

406 Wrapper around `_all_annotations` that filters away any keys in _except. 

407 

408 It also flattens the ChainMap to a regular dict. 

409 """ 

410 if _except is None: 

411 _except = set() 

412 

413 _all = _all_annotations(cls) 

414 return {k: v for k, v in _all.items() if k not in _except} 

415 

416 

417def check_and_convert_data( 

418 cls: typing.Type[C], 

419 data: dict[str, typing.Any], 

420 _except: typing.Iterable[str], 

421 strict: bool = True, 

422 convert_types: bool = False, 

423) -> dict[str, typing.Any]: 

424 """ 

425 Based on class annotations, this prepares the data for `load_into_recurse`. 

426 

427 1. convert config-keys to python compatible config_keys 

428 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

429 3. ensures the annotated types match the actual types after loading the config file. 

430 """ 

431 annotations = all_annotations(cls, _except=_except) 

432 

433 to_load = convert_config(data) 

434 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types) 

435 if strict: 

436 to_load = ensure_types(to_load, annotations, convert_types=convert_types) 

437 

438 return to_load 

439 

440 

441T_init_list = list[typing.Any] 

442T_init_dict = dict[str, typing.Any] 

443T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

444 

445 

446@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

447def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

448 """ 

449 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

450 """ 

451 if not init: 

452 return [], {} 

453 

454 args: T_init_list = [] 

455 kwargs: T_init_dict = {} 

456 match init: 

457 case (args, kwargs): 

458 return args, kwargs 

459 case [*args]: 

460 return args, {} 

461 case {**kwargs}: 

462 return [], kwargs 

463 case _: 

464 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

465 

466 

467def _load_into_recurse( 

468 cls: typing.Type[C], 

469 data: dict[str, typing.Any], 

470 init: T_init = None, 

471 strict: bool = True, 

472 convert_types: bool = False, 

473) -> C: 

474 """ 

475 Loads an instance of `cls` filled with `data`. 

476 

477 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

478 `init` can be used to optionally pass extra __init__ arguments. \ 

479 NOTE: This will overwrite a config key with the same name! 

480 """ 

481 init_args, init_kwargs = _split_init(init) 

482 

483 if dc.is_dataclass(cls): 

484 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types) 

485 if init: 

486 raise ValueError("Init is not allowed for dataclasses!") 

487 

488 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

489 inst = typing.cast(C, cls(**to_load)) 

490 else: 

491 inst = cls(*init_args, **init_kwargs) 

492 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types) 

493 inst.__dict__.update(**to_load) 

494 

495 return inst 

496 

497 

498def _load_into_instance( 

499 inst: C, 

500 cls: typing.Type[C], 

501 data: dict[str, typing.Any], 

502 init: T_init = None, 

503 strict: bool = True, 

504 convert_types: bool = False, 

505) -> C: 

506 """ 

507 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

508 and thus does not support init. 

509 

510 """ 

511 if init is not None: 

512 raise ValueError("Can not init an existing instance!") 

513 

514 existing_data = inst.__dict__ 

515 

516 to_load = check_and_convert_data( 

517 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types 

518 ) 

519 

520 inst.__dict__.update(**to_load) 

521 

522 return inst 

523 

524 

525def load_into_class( 

526 cls: typing.Type[C], 

527 data: T_data, 

528 /, 

529 key: str = None, 

530 init: T_init = None, 

531 strict: bool = True, 

532 lower_keys: bool = False, 

533 convert_types: bool = False, 

534) -> C: 

535 """ 

536 Shortcut for _load_data + load_into_recurse. 

537 """ 

538 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys) 

539 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types) 

540 

541 

542def load_into_instance( 

543 inst: C, 

544 data: T_data, 

545 /, 

546 key: str = None, 

547 init: T_init = None, 

548 strict: bool = True, 

549 lower_keys: bool = False, 

550 convert_types: bool = False, 

551) -> C: 

552 """ 

553 Shortcut for _load_data + load_into_existing. 

554 """ 

555 cls = inst.__class__ 

556 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys) 

557 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types) 

558 

559 

560def load_into( 

561 cls: typing.Type[C], 

562 data: T_data = None, 

563 /, 

564 key: str = None, 

565 init: T_init = None, 

566 strict: bool = True, 

567 lower_keys: bool = False, 

568 convert_types: bool = False, 

569) -> C: 

570 """ 

571 Load your config into a class (instance). 

572 

573 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

574 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

575 

576 Args: 

577 cls: either a class or an existing instance of that class. 

578 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

579 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

580 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

581 strict: enable type checks or allow anything? 

582 lower_keys: should the config keys be lowercased? (for .env) 

583 convert_types: should the types be converted to the annotated type if not yet matching? (for .env) 

584 

585 """ 

586 if not isinstance(cls, type): 

587 # would not be supported according to mypy, but you can still load_into(instance) 

588 return load_into_instance( 

589 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

590 ) 

591 

592 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

593 # _cls = typing.cast(typing.Type[C], cls) 

594 return load_into_class( 

595 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

596 )