Coverage for src/configuraptor/core.py: 100%
184 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-26 13:54 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-26 13:54 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import math
7import types
8import typing
9import warnings
10from collections import ChainMap
11from pathlib import Path
13from typeguard import TypeCheckError
14from typeguard import check_type as _check_type
16from . import loaders
17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey
18from .helpers import camel_to_snake
19from .postpone import Postponed
21# T is a reusable typevar
22T = typing.TypeVar("T")
23# t_typelike is anything that can be type hinted
24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union
25# t_data is anything that can be fed to _load_data
26T_data = str | Path | dict[str, typing.Any]
27# c = a config class instance, can be any (user-defined) class
28C = typing.TypeVar("C")
29# type c is a config class
30Type_C = typing.Type[C]
33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
34 """
35 If a key contains a dot, traverse the raw dict until the right key was found.
37 Example:
38 key = some.nested.key
39 raw = {"some": {"nested": {"key": {"with": "data"}}}}
40 -> {"with": "data"}
41 """
42 parts = key.split(".")
43 while parts:
44 raw = raw[parts.pop(0)]
46 return raw
49def _guess_key(clsname: str) -> str:
50 """
51 If no key is manually defined for `load_into`, \
52 the class' name is converted to snake_case to use as the default key.
53 """
54 return camel_to_snake(clsname)
57def __load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:
58 """
59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
61 E.g. class Tool will be mapped to key tool.
62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
63 """
64 if isinstance(data, str):
65 data = Path(data)
66 if isinstance(data, Path):
67 with data.open("rb") as f:
68 loader = loaders.get(data.suffix)
69 data = loader(f)
71 if not data:
72 return {}
74 if key is None:
75 # try to guess key by grabbing the first one or using the class name
76 if len(data) == 1:
77 key = list(data.keys())[0]
78 elif classname is not None:
79 key = _guess_key(classname)
81 if key:
82 data = _data_for_nested_key(key, data)
84 if not data:
85 raise ValueError("No data found!")
87 if not isinstance(data, dict):
88 raise ValueError("Data is not a dict!")
90 return data
93def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:
94 """
95 Wrapper around __load_data that retries with key="" if anything goes wrong.
96 """
97 try:
98 return __load_data(data, key, classname)
99 except Exception as e:
100 if key != "":
101 return __load_data(data, "", classname)
102 else: # pragma: no cover
103 warnings.warn(f"Data could not be loaded: {e}", source=e)
104 # key already was "", just return data!
105 # (will probably not happen but fallback)
106 return {}
109def check_type(value: typing.Any, expected_type: T_typelike) -> bool:
110 """
111 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).
113 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError
114 """
115 try:
116 _check_type(value, expected_type)
117 return True
118 except TypeCheckError:
119 return False
122def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:
123 """
124 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
126 If an annotated key in missing from data, it will be filled with None for convenience.
128 TODO: python 3.11 exception groups to throw multiple errors at once!
129 """
130 # custom object to use instead of None, since typing.Optional can be None!
131 # cast to T to make mypy happy
132 notfound = typing.cast(T, object())
133 postponed = Postponed()
135 final: dict[str, T | None] = {}
136 for key, _type in annotations.items():
137 compare = data.get(key, notfound)
138 if compare is notfound: # pragma: nocover
139 warnings.warn(
140 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
141 )
142 # skip!
143 continue
145 if compare is postponed:
146 # don't do anything with this item!
147 continue
149 if not check_type(compare, _type):
150 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
152 final[key] = compare
154 return final
157def convert_config(items: dict[str, T]) -> dict[str, T]:
158 """
159 Converts the config dict (from toml) or 'overwrites' dict in two ways.
161 1. removes any items where the value is None, since in that case the default should be used;
162 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
163 """
164 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
167Type = typing.Type[typing.Any]
168T_Type = typing.TypeVar("T_Type", bound=Type)
171def is_builtin_type(_type: Type) -> bool:
172 """
173 Returns whether _type is one of the builtin types.
174 """
175 return _type.__module__ in ("__builtin__", "builtins")
178# def is_builtin_class_instance(obj: typing.Any) -> bool:
179# return is_builtin_type(obj.__class__)
182def is_from_types_or_typing(_type: Type) -> bool:
183 """
184 Returns whether _type is one of the stlib typing/types types.
186 e.g. types.UnionType or typing.Union
187 """
188 return _type.__module__ in ("types", "typing")
191def is_from_other_toml_supported_module(_type: Type) -> bool:
192 """
193 Besides builtins, toml also supports 'datetime' and 'math' types, \
194 so this returns whether _type is a type from these stdlib modules.
195 """
196 return _type.__module__ in ("datetime", "math")
199def is_parameterized(_type: Type) -> bool:
200 """
201 Returns whether _type is a parameterized type.
203 Examples:
204 list[str] -> True
205 str -> False
206 """
207 return typing.get_origin(_type) is not None
210def is_custom_class(_type: Type) -> bool:
211 """
212 Tries to guess if _type is a builtin or a custom (user-defined) class.
214 Other logic in this module depends on knowing that.
215 """
216 return (
217 type(_type) is type
218 and not is_builtin_type(_type)
219 and not is_from_other_toml_supported_module(_type)
220 and not is_from_types_or_typing(_type)
221 )
224def instance_of_custom_class(var: typing.Any) -> bool:
225 """
226 Calls `is_custom_class` on an instance of a (possibly custom) class.
227 """
228 return is_custom_class(var.__class__)
231def is_optional(_type: Type | typing.Any) -> bool:
232 """
233 Tries to guess if _type could be optional.
235 Examples:
236 None -> True
237 NoneType -> True
238 typing.Union[str, None] -> True
239 str | None -> True
240 list[str | None] -> False
241 list[str] -> False
242 """
243 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):
244 # e.g. list[str]
245 # will crash issubclass to test it first here
246 return False
248 return (
249 _type is None
250 or types.NoneType in typing.get_args(_type) # union with Nonetype
251 or issubclass(types.NoneType, _type)
252 or issubclass(types.NoneType, type(_type)) # no type # Nonetype
253 )
256def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:
257 """
258 Get Field info for a dataclass cls.
259 """
260 fields = getattr(cls, "__dataclass_fields__", {})
261 return fields.get(key)
264def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:
265 """
266 For all annotations (recursively gathered from parents with `all_annotations`), \
267 try to resolve the tree of annotations.
269 Uses `load_into_recurse`, not itself directly.
271 Example:
272 class First:
273 key: str
275 class Second:
276 other: First
278 # step 1
279 cls = Second
280 data = {"second": {"other": {"key": "anything"}}}
281 annotations: {"other": First}
283 # step 1.5
284 data = {"other": {"key": "anything"}
285 annotations: {"other": First}
287 # step 2
288 cls = First
289 data = {"key": "anything"}
290 annotations: {"key": str}
293 TODO: python 3.11 exception groups to throw multiple errors at once!
294 """
295 updated = {}
297 for _key, _type in annotations.items():
298 if _key in data:
299 value: typing.Any = data[_key] # value can change so define it as any instead of T
300 if is_parameterized(_type):
301 origin = typing.get_origin(_type)
302 arguments = typing.get_args(_type)
303 if origin is list and arguments and is_custom_class(arguments[0]):
304 subtype = arguments[0]
305 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]
307 elif origin is dict and arguments and is_custom_class(arguments[1]):
308 # e.g. dict[str, Point]
309 subkeytype, subvaluetype = arguments
310 # subkey(type) is not a custom class, so don't try to convert it:
311 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}
312 # elif origin is dict:
313 # keep data the same
314 elif origin is typing.Union and arguments:
315 for arg in arguments:
316 if is_custom_class(arg):
317 value = _load_into_recurse(arg, value)
318 else:
319 # print(_type, arg, value)
320 ...
322 # todo: other parameterized/unions/typing.Optional
324 elif is_custom_class(_type):
325 # type must be C (custom class) at this point
326 value = _load_into_recurse(
327 # make mypy and pycharm happy by telling it _type is of type C...
328 # actually just passing _type as first arg!
329 typing.cast(Type_C[typing.Any], _type),
330 value,
331 )
333 elif _key in cls.__dict__:
334 # property has default, use that instead.
335 value = cls.__dict__[_key]
336 elif is_optional(_type):
337 # type is optional and not found in __dict__ -> default is None
338 value = None
339 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
340 # could have a default factory
341 # todo: do something with field.default?
342 value = field.default_factory()
343 else:
344 raise ConfigErrorMissingKey(_key, cls, _type)
346 updated[_key] = value
348 return updated
351def _all_annotations(cls: Type) -> ChainMap[str, Type]:
352 """
353 Returns a dictionary-like ChainMap that includes annotations for all \
354 attributes defined in cls or inherited from superclasses.
355 """
356 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
359def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:
360 """
361 Wrapper around `_all_annotations` that filters away any keys in _except.
363 It also flattens the ChainMap to a regular dict.
364 """
365 if _except is None:
366 _except = set()
368 _all = _all_annotations(cls)
369 return {k: v for k, v in _all.items() if k not in _except}
372def check_and_convert_data(
373 cls: typing.Type[C],
374 data: dict[str, typing.Any],
375 _except: typing.Iterable[str],
376 strict: bool = True,
377) -> dict[str, typing.Any]:
378 """
379 Based on class annotations, this prepares the data for `load_into_recurse`.
381 1. convert config-keys to python compatible config_keys
382 2. loads custom class type annotations with the same logic (see also `load_recursive`)
383 3. ensures the annotated types match the actual types after loading the config file.
384 """
385 annotations = all_annotations(cls, _except=_except)
387 to_load = convert_config(data)
388 to_load = load_recursive(cls, to_load, annotations)
389 if strict:
390 to_load = ensure_types(to_load, annotations)
392 return to_load
395T_init_list = list[typing.Any]
396T_init_dict = dict[str, typing.Any]
397T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None
400@typing.no_type_check # (mypy doesn't understand 'match' fully yet)
401def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:
402 """
403 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.
404 """
405 if not init:
406 return [], {}
408 args: T_init_list = []
409 kwargs: T_init_dict = {}
410 match init:
411 case (args, kwargs):
412 return args, kwargs
413 case [*args]:
414 return args, {}
415 case {**kwargs}:
416 return [], kwargs
417 case _:
418 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")
421def _load_into_recurse(
422 cls: typing.Type[C],
423 data: dict[str, typing.Any],
424 init: T_init = None,
425 strict: bool = True,
426) -> C:
427 """
428 Loads an instance of `cls` filled with `data`.
430 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
431 `init` can be used to optionally pass extra __init__ arguments. \
432 NOTE: This will overwrite a config key with the same name!
433 """
434 init_args, init_kwargs = _split_init(init)
436 if dc.is_dataclass(cls):
437 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict)
438 if init:
439 raise ValueError("Init is not allowed for dataclasses!")
441 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
442 inst = typing.cast(C, cls(**to_load))
443 else:
444 inst = cls(*init_args, **init_kwargs)
445 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict)
446 inst.__dict__.update(**to_load)
448 return inst
451def _load_into_instance(
452 inst: C,
453 cls: typing.Type[C],
454 data: dict[str, typing.Any],
455 init: T_init = None,
456 strict: bool = True,
457) -> C:
458 """
459 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
460 and thus does not support init.
462 """
463 if init is not None:
464 raise ValueError("Can not init an existing instance!")
466 existing_data = inst.__dict__
468 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict)
470 inst.__dict__.update(**to_load)
472 return inst
475def load_into_class(
476 cls: typing.Type[C],
477 data: T_data,
478 /,
479 key: str = None,
480 init: T_init = None,
481 strict: bool = True,
482) -> C:
483 """
484 Shortcut for _load_data + load_into_recurse.
485 """
486 to_load = _load_data(data, key, cls.__name__)
487 return _load_into_recurse(cls, to_load, init=init, strict=strict)
490def load_into_instance(
491 inst: C,
492 data: T_data,
493 /,
494 key: str = None,
495 init: T_init = None,
496 strict: bool = True,
497) -> C:
498 """
499 Shortcut for _load_data + load_into_existing.
500 """
501 cls = inst.__class__
502 to_load = _load_data(data, key, cls.__name__)
503 return _load_into_instance(inst, cls, to_load, init=init, strict=strict)
506def load_into(
507 cls: typing.Type[C],
508 data: T_data,
509 /,
510 key: str = None,
511 init: T_init = None,
512 strict: bool = True,
513) -> C:
514 """
515 Load your config into a class (instance).
517 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
518 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
520 Args:
521 cls: either a class or an existing instance of that class.
522 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
523 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
524 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
525 strict: enable type checks or allow anything?
527 """
528 if not isinstance(cls, type):
529 # would not be supported according to mypy, but you can still load_into(instance)
530 return load_into_instance(cls, data, key=key, init=init, strict=strict)
532 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
533 # _cls = typing.cast(typing.Type[C], cls)
534 return load_into_class(cls, data, key=key, init=init, strict=strict)