Coverage for src/configuraptor/core.py: 100%
205 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 15:07 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 15:07 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import math
7import types
8import typing
9import warnings
10from collections import ChainMap
11from pathlib import Path
13from typeguard import TypeCheckError
14from typeguard import check_type as _check_type
16from . import loaders
17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey, ConfigErrorCouldNotConvert
18from .helpers import camel_to_snake
19from .postpone import Postponed
21# T is a reusable typevar
22T = typing.TypeVar("T")
23# t_typelike is anything that can be type hinted
24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union
25# t_data is anything that can be fed to _load_data
26T_data = str | Path | dict[str, typing.Any]
27# c = a config class instance, can be any (user-defined) class
28C = typing.TypeVar("C")
29# type c is a config class
30Type_C = typing.Type[C]
33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
34 """
35 If a key contains a dot, traverse the raw dict until the right key was found.
37 Example:
38 key = some.nested.key
39 raw = {"some": {"nested": {"key": {"with": "data"}}}}
40 -> {"with": "data"}
41 """
42 parts = key.split(".")
43 while parts:
44 raw = raw[parts.pop(0)]
46 return raw
49def _guess_key(clsname: str) -> str:
50 """
51 If no key is manually defined for `load_into`, \
52 the class' name is converted to snake_case to use as the default key.
53 """
54 return camel_to_snake(clsname)
57def __load_data(
58 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False
59) -> dict[str, typing.Any]:
60 """
61 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
63 E.g. class Tool will be mapped to key tool.
64 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
65 """
66 if isinstance(data, str):
67 data = Path(data)
68 if isinstance(data, Path):
69 with data.open("rb") as f:
70 loader = loaders.get(data.suffix or data.name)
71 data = loader(f)
73 if not data:
74 return {}
76 if key is None:
77 # try to guess key by grabbing the first one or using the class name
78 if len(data) == 1:
79 key = list(data.keys())[0]
80 elif classname is not None:
81 key = _guess_key(classname)
83 if key:
84 data = _data_for_nested_key(key, data)
86 if not data:
87 raise ValueError("No data found!")
89 if not isinstance(data, dict):
90 raise ValueError("Data is not a dict!")
92 if lower_keys:
93 data = {k.lower(): v for k, v in data.items()}
95 return data
98def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]:
99 """
100 Wrapper around __load_data that retries with key="" if anything goes wrong.
101 """
102 try:
103 return __load_data(data, key, classname, lower_keys=lower_keys)
104 except Exception as e:
105 if key != "":
106 return __load_data(data, "", classname, lower_keys=lower_keys)
107 else: # pragma: no cover
108 warnings.warn(f"Data could not be loaded: {e}", source=e)
109 # key already was "", just return data!
110 # (will probably not happen but fallback)
111 return {}
114def check_type(value: typing.Any, expected_type: T_typelike) -> bool:
115 """
116 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).
118 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError
119 """
120 try:
121 _check_type(value, expected_type)
122 return True
123 except TypeCheckError:
124 return False
127F = typing.TypeVar("F")
130def str_to_bool(value: str) -> bool:
131 """
132 Used by convert_between, usually for .env loads.
134 Example:
135 SOME_VALUE=TRUE -> True
136 SOME_VALUE=1 -> True
137 SOME_VALUE=Yes -> True
139 SOME_VALUE -> None
140 SOME_VALUE=NOpe -> False
142 SOME_VALUE=Unrelated -> Error
143 """
144 if not value:
145 return False
147 first_letter = value[0].lower()
148 # yes, true, 1
149 if first_letter in {"y", "t", "1"}:
150 return True
151 elif first_letter in {"n", "f", "0"}:
152 return False
153 else:
154 raise ValueError("Not booly.")
157def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:
158 """
159 Convert a value between types.
160 """
161 if from_type is str and to_type is bool:
162 return str_to_bool(from_value)
163 # default: just convert type:
164 return to_type(from_value)
167def ensure_types(
168 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False
169) -> dict[str, T | None]:
170 """
171 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
173 If an annotated key in missing from data, it will be filled with None for convenience.
175 TODO: python 3.11 exception groups to throw multiple errors at once!
176 """
177 # custom object to use instead of None, since typing.Optional can be None!
178 # cast to T to make mypy happy
179 notfound = typing.cast(T, object())
180 postponed = Postponed()
182 final: dict[str, T | None] = {}
183 for key, _type in annotations.items():
184 compare = data.get(key, notfound)
185 if compare is notfound: # pragma: nocover
186 warnings.warn(
187 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
188 )
189 # skip!
190 continue
192 if compare is postponed:
193 # don't do anything with this item!
194 continue
196 if not check_type(compare, _type):
197 if convert_types:
198 try:
199 compare = convert_between(compare, type(compare), _type)
200 except (TypeError, ValueError) as e:
201 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e
202 else:
203 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
205 final[key] = compare
207 return final
210def convert_config(items: dict[str, T]) -> dict[str, T]:
211 """
212 Converts the config dict (from toml) or 'overwrites' dict in two ways.
214 1. removes any items where the value is None, since in that case the default should be used;
215 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
216 """
217 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
220Type = typing.Type[typing.Any]
221T_Type = typing.TypeVar("T_Type", bound=Type)
224def is_builtin_type(_type: Type) -> bool:
225 """
226 Returns whether _type is one of the builtin types.
227 """
228 return _type.__module__ in ("__builtin__", "builtins")
231# def is_builtin_class_instance(obj: typing.Any) -> bool:
232# return is_builtin_type(obj.__class__)
235def is_from_types_or_typing(_type: Type) -> bool:
236 """
237 Returns whether _type is one of the stlib typing/types types.
239 e.g. types.UnionType or typing.Union
240 """
241 return _type.__module__ in ("types", "typing")
244def is_from_other_toml_supported_module(_type: Type) -> bool:
245 """
246 Besides builtins, toml also supports 'datetime' and 'math' types, \
247 so this returns whether _type is a type from these stdlib modules.
248 """
249 return _type.__module__ in ("datetime", "math")
252def is_parameterized(_type: Type) -> bool:
253 """
254 Returns whether _type is a parameterized type.
256 Examples:
257 list[str] -> True
258 str -> False
259 """
260 return typing.get_origin(_type) is not None
263def is_custom_class(_type: Type) -> bool:
264 """
265 Tries to guess if _type is a builtin or a custom (user-defined) class.
267 Other logic in this module depends on knowing that.
268 """
269 return (
270 type(_type) is type
271 and not is_builtin_type(_type)
272 and not is_from_other_toml_supported_module(_type)
273 and not is_from_types_or_typing(_type)
274 )
277def instance_of_custom_class(var: typing.Any) -> bool:
278 """
279 Calls `is_custom_class` on an instance of a (possibly custom) class.
280 """
281 return is_custom_class(var.__class__)
284def is_optional(_type: Type | typing.Any) -> bool:
285 """
286 Tries to guess if _type could be optional.
288 Examples:
289 None -> True
290 NoneType -> True
291 typing.Union[str, None] -> True
292 str | None -> True
293 list[str | None] -> False
294 list[str] -> False
295 """
296 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):
297 # e.g. list[str]
298 # will crash issubclass to test it first here
299 return False
301 return (
302 _type is None
303 or types.NoneType in typing.get_args(_type) # union with Nonetype
304 or issubclass(types.NoneType, _type)
305 or issubclass(types.NoneType, type(_type)) # no type # Nonetype
306 )
309def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:
310 """
311 Get Field info for a dataclass cls.
312 """
313 fields = getattr(cls, "__dataclass_fields__", {})
314 return fields.get(key)
317def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:
318 """
319 For all annotations (recursively gathered from parents with `all_annotations`), \
320 try to resolve the tree of annotations.
322 Uses `load_into_recurse`, not itself directly.
324 Example:
325 class First:
326 key: str
328 class Second:
329 other: First
331 # step 1
332 cls = Second
333 data = {"second": {"other": {"key": "anything"}}}
334 annotations: {"other": First}
336 # step 1.5
337 data = {"other": {"key": "anything"}
338 annotations: {"other": First}
340 # step 2
341 cls = First
342 data = {"key": "anything"}
343 annotations: {"key": str}
346 TODO: python 3.11 exception groups to throw multiple errors at once!
347 """
348 updated = {}
350 for _key, _type in annotations.items():
351 if _key in data:
352 value: typing.Any = data[_key] # value can change so define it as any instead of T
353 if is_parameterized(_type):
354 origin = typing.get_origin(_type)
355 arguments = typing.get_args(_type)
356 if origin is list and arguments and is_custom_class(arguments[0]):
357 subtype = arguments[0]
358 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]
360 elif origin is dict and arguments and is_custom_class(arguments[1]):
361 # e.g. dict[str, Point]
362 subkeytype, subvaluetype = arguments
363 # subkey(type) is not a custom class, so don't try to convert it:
364 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}
365 # elif origin is dict:
366 # keep data the same
367 elif origin is typing.Union and arguments:
368 for arg in arguments:
369 if is_custom_class(arg):
370 value = _load_into_recurse(arg, value)
371 else:
372 # print(_type, arg, value)
373 ...
375 # todo: other parameterized/unions/typing.Optional
377 elif is_custom_class(_type):
378 # type must be C (custom class) at this point
379 value = _load_into_recurse(
380 # make mypy and pycharm happy by telling it _type is of type C...
381 # actually just passing _type as first arg!
382 typing.cast(Type_C[typing.Any], _type),
383 value,
384 )
386 elif _key in cls.__dict__:
387 # property has default, use that instead.
388 value = cls.__dict__[_key]
389 elif is_optional(_type):
390 # type is optional and not found in __dict__ -> default is None
391 value = None
392 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
393 # could have a default factory
394 # todo: do something with field.default?
395 value = field.default_factory()
396 else:
397 raise ConfigErrorMissingKey(_key, cls, _type)
399 updated[_key] = value
401 return updated
404def _all_annotations(cls: Type) -> ChainMap[str, Type]:
405 """
406 Returns a dictionary-like ChainMap that includes annotations for all \
407 attributes defined in cls or inherited from superclasses.
408 """
409 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
412def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, type[object]]:
413 """
414 Wrapper around `_all_annotations` that filters away any keys in _except.
416 It also flattens the ChainMap to a regular dict.
417 """
418 if _except is None:
419 _except = set()
421 _all = _all_annotations(cls)
422 return {k: v for k, v in _all.items() if k not in _except}
425def check_and_convert_data(
426 cls: typing.Type[C],
427 data: dict[str, typing.Any],
428 _except: typing.Iterable[str],
429 strict: bool = True,
430 convert_types: bool = False,
431) -> dict[str, typing.Any]:
432 """
433 Based on class annotations, this prepares the data for `load_into_recurse`.
435 1. convert config-keys to python compatible config_keys
436 2. loads custom class type annotations with the same logic (see also `load_recursive`)
437 3. ensures the annotated types match the actual types after loading the config file.
438 """
439 annotations = all_annotations(cls, _except=_except)
441 to_load = convert_config(data)
442 to_load = load_recursive(cls, to_load, annotations)
443 if strict:
444 to_load = ensure_types(to_load, annotations, convert_types=convert_types)
446 return to_load
449T_init_list = list[typing.Any]
450T_init_dict = dict[str, typing.Any]
451T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None
454@typing.no_type_check # (mypy doesn't understand 'match' fully yet)
455def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:
456 """
457 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.
458 """
459 if not init:
460 return [], {}
462 args: T_init_list = []
463 kwargs: T_init_dict = {}
464 match init:
465 case (args, kwargs):
466 return args, kwargs
467 case [*args]:
468 return args, {}
469 case {**kwargs}:
470 return [], kwargs
471 case _:
472 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")
475def _load_into_recurse(
476 cls: typing.Type[C],
477 data: dict[str, typing.Any],
478 init: T_init = None,
479 strict: bool = True,
480 convert_types: bool = False,
481) -> C:
482 """
483 Loads an instance of `cls` filled with `data`.
485 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
486 `init` can be used to optionally pass extra __init__ arguments. \
487 NOTE: This will overwrite a config key with the same name!
488 """
489 init_args, init_kwargs = _split_init(init)
491 if dc.is_dataclass(cls):
492 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)
493 if init:
494 raise ValueError("Init is not allowed for dataclasses!")
496 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
497 inst = typing.cast(C, cls(**to_load))
498 else:
499 inst = cls(*init_args, **init_kwargs)
500 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)
501 inst.__dict__.update(**to_load)
503 return inst
506def _load_into_instance(
507 inst: C,
508 cls: typing.Type[C],
509 data: dict[str, typing.Any],
510 init: T_init = None,
511 strict: bool = True,
512 convert_types: bool = False,
513) -> C:
514 """
515 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
516 and thus does not support init.
518 """
519 if init is not None:
520 raise ValueError("Can not init an existing instance!")
522 existing_data = inst.__dict__
524 to_load = check_and_convert_data(
525 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types
526 )
528 inst.__dict__.update(**to_load)
530 return inst
533def load_into_class(
534 cls: typing.Type[C],
535 data: T_data,
536 /,
537 key: str = None,
538 init: T_init = None,
539 strict: bool = True,
540 lower_keys: bool = False,
541 convert_types: bool = False,
542) -> C:
543 """
544 Shortcut for _load_data + load_into_recurse.
545 """
546 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)
547 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)
550def load_into_instance(
551 inst: C,
552 data: T_data,
553 /,
554 key: str = None,
555 init: T_init = None,
556 strict: bool = True,
557 lower_keys: bool = False,
558 convert_types: bool = False,
559) -> C:
560 """
561 Shortcut for _load_data + load_into_existing.
562 """
563 cls = inst.__class__
564 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)
565 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)
568def load_into(
569 cls: typing.Type[C],
570 data: T_data,
571 /,
572 key: str = None,
573 init: T_init = None,
574 strict: bool = True,
575 lower_keys: bool = False,
576 convert_types: bool = False,
577) -> C:
578 """
579 Load your config into a class (instance).
581 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
582 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
584 Args:
585 cls: either a class or an existing instance of that class.
586 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
587 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
588 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
589 strict: enable type checks or allow anything?
591 """
592 if not isinstance(cls, type):
593 # would not be supported according to mypy, but you can still load_into(instance)
594 return load_into_instance(
595 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
596 )
598 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
599 # _cls = typing.cast(typing.Type[C], cls)
600 return load_into_class(
601 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
602 )