Coverage for src/configuraptor/core.py: 100%
144 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 11:44 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 11:44 +0200
1"""
2Contains most of the loading logic.
3"""
5import types
6import typing
7import warnings
8from collections import ChainMap
9from dataclasses import is_dataclass
10from pathlib import Path
12from typeguard import TypeCheckError
13from typeguard import check_type as _check_type
15from . import loaders
16from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey
17from .helpers import camel_to_snake
19# T is a reusable typevar
20T = typing.TypeVar("T")
21# t_typelike is anything that can be type hinted
22T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union
23# t_data is anything that can be fed to _load_data
24T_data = str | Path | dict[str, typing.Any]
25# c = a config class instance, can be any (user-defined) class
26C = typing.TypeVar("C")
27# type c is a config class
28Type_C = typing.Type[C]
31def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
32 """
33 If a key contains a dot, traverse the raw dict until the right key was found.
35 Example:
36 key = some.nested.key
37 raw = {"some": {"nested": {"key": {"with": "data"}}}}
38 -> {"with": "data"}
39 """
40 parts = key.split(".")
41 while parts:
42 raw = raw[parts.pop(0)]
44 return raw
47def _guess_key(clsname: str) -> str:
48 """
49 If no key is manually defined for `load_into`, \
50 the class' name is converted to snake_case to use as the default key.
51 """
52 return camel_to_snake(clsname)
55def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:
56 """
57 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
59 E.g. class Tool will be mapped to key tool.
60 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
61 """
62 if isinstance(data, str):
63 data = Path(data)
64 if isinstance(data, Path):
65 # todo: more than toml
66 with data.open("rb") as f:
67 data = loaders.toml(f)
69 if not data:
70 return {}
72 if key is None:
73 # try to guess key by grabbing the first one or using the class name
74 if len(data) == 1:
75 key = list(data.keys())[0]
76 elif classname is not None:
77 key = _guess_key(classname)
79 if key:
80 return _data_for_nested_key(key, data)
81 else:
82 # no key found, just return all data
83 return data
86def check_type(value: typing.Any, expected_type: T_typelike) -> bool:
87 """
88 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).
90 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError
91 """
92 try:
93 _check_type(value, expected_type)
94 return True
95 except TypeCheckError:
96 return False
99def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:
100 """
101 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
103 If an annotated key in missing from data, it will be filled with None for convenience.
104 """
105 # custom object to use instead of None, since typing.Optional can be None!
106 # cast to T to make mypy happy
107 notfound = typing.cast(T, object())
109 final: dict[str, T | None] = {}
110 for key, _type in annotations.items():
111 compare = data.get(key, notfound)
112 if compare is notfound: # pragma: nocover
113 warnings.warn(
114 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
115 )
116 # skip!
117 continue
118 if not check_type(compare, _type):
119 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
121 final[key] = compare
122 return final
125def convert_config(items: dict[str, T]) -> dict[str, T]:
126 """
127 Converts the config dict (from toml) or 'overwrites' dict in two ways.
129 1. removes any items where the value is None, since in that case the default should be used;
130 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
131 """
132 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
135Type = typing.Type[typing.Any]
136T_Type = typing.TypeVar("T_Type", bound=Type)
139def is_builtin_type(_type: Type) -> bool:
140 """
141 Returns whether _type is one of the builtin types.
142 """
143 return _type.__module__ in ("__builtin__", "builtins")
146# def is_builtin_class_instance(obj: typing.Any) -> bool:
147# return is_builtin_type(obj.__class__)
150def is_from_types_or_typing(_type: Type) -> bool:
151 """
152 Returns whether _type is one of the stlib typing/types types.
154 e.g. types.UnionType or typing.Union
155 """
156 return _type.__module__ in ("types", "typing")
159def is_from_other_toml_supported_module(_type: Type) -> bool:
160 """
161 Besides builtins, toml also supports 'datetime' and 'math' types, \
162 so this returns whether _type is a type from these stdlib modules.
163 """
164 return _type.__module__ in ("datetime", "math")
167def is_parameterized(_type: Type) -> bool:
168 """
169 Returns whether _type is a parameterized type.
171 Examples:
172 list[str] -> True
173 str -> False
174 """
175 return typing.get_origin(_type) is not None
178def is_custom_class(_type: Type) -> bool:
179 """
180 Tries to guess if _type is a builtin or a custom (user-defined) class.
182 Other logic in this module depends on knowing that.
183 """
184 return (
185 type(_type) is type
186 and not is_builtin_type(_type)
187 and not is_from_other_toml_supported_module(_type)
188 and not is_from_types_or_typing(_type)
189 )
192def is_optional(_type: Type | None) -> bool:
193 """
194 Tries to guess if _type could be optional.
196 Examples:
197 None -> True
198 NoneType -> True
199 typing.Union[str, None] -> True
200 str | None -> True
201 list[str | None] -> False
202 list[str] -> False
203 """
204 if _type and is_parameterized(_type) and typing.get_origin(_type) in (dict, list):
205 # e.g. list[str]
206 # will crash issubclass to test it first here
207 return False
209 return (
210 _type is None
211 or issubclass(types.NoneType, _type)
212 or issubclass(types.NoneType, type(_type)) # no type # Nonetype
213 or type(None) in typing.get_args(_type) # union with Nonetype
214 )
217def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:
218 """
219 For all annotations (recursively gathered from parents with `all_annotations`), \
220 try to resolve the tree of annotations.
222 Uses `load_into_recurse`, not itself directly.
224 Example:
225 class First:
226 key: str
228 class Second:
229 other: First
231 # step 1
232 cls = Second
233 data = {"second": {"other": {"key": "anything"}}}
234 annotations: {"other": First}
236 # step 1.5
237 data = {"other": {"key": "anything"}
238 annotations: {"other": First}
240 # step 2
241 cls = First
242 data = {"key": "anything"}
243 annotations: {"key": str}
245 """
246 updated = {}
247 for _key, _type in annotations.items():
248 if _key in data:
249 value: typing.Any = data[_key] # value can change so define it as any instead of T
250 if is_parameterized(_type):
251 origin = typing.get_origin(_type)
252 arguments = typing.get_args(_type)
253 if origin is list and arguments and is_custom_class(arguments[0]):
254 subtype = arguments[0]
255 value = [load_into_recurse(subtype, subvalue) for subvalue in value]
257 elif origin is dict and arguments and is_custom_class(arguments[1]):
258 # e.g. dict[str, Point]
259 subkeytype, subvaluetype = arguments
260 # subkey(type) is not a custom class, so don't try to convert it:
261 value = {subkey: load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}
262 # elif origin is dict:
263 # keep data the same
264 elif origin is typing.Union and arguments:
265 for arg in arguments:
266 if is_custom_class(arg):
267 value = load_into_recurse(arg, value)
268 else:
269 # print(_type, arg, value)
270 ...
272 # todo: other parameterized/unions/typing.Optional
274 elif is_custom_class(_type):
275 # type must be C (custom class) at this point
276 value = load_into_recurse(
277 # make mypy and pycharm happy by telling it _type is of type C...
278 # actually just passing _type as first arg!
279 typing.cast(Type_C[typing.Any], _type),
280 value,
281 )
283 elif _key in cls.__dict__:
284 # property has default, use that instead.
285 value = cls.__dict__[_key]
286 elif is_optional(_type):
287 # type is optional and not found in __dict__ -> default is None
288 value = None
289 else:
290 # todo: exception group?
291 raise ConfigErrorMissingKey(_key, cls, _type)
293 updated[_key] = value
295 return updated
298def _all_annotations(cls: Type) -> ChainMap[str, Type]:
299 """
300 Returns a dictionary-like ChainMap that includes annotations for all \
301 attributes defined in cls or inherited from superclasses.
302 """
303 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
306def all_annotations(cls: Type, _except: typing.Iterable[str]) -> dict[str, Type]:
307 """
308 Wrapper around `_all_annotations` that filters away any keys in _except.
310 It also flattens the ChainMap to a regular dict.
311 """
312 _all = _all_annotations(cls)
313 return {k: v for k, v in _all.items() if k not in _except}
316def _check_and_convert_data(
317 cls: typing.Type[C],
318 data: dict[str, typing.Any],
319 _except: typing.Iterable[str],
320) -> dict[str, typing.Any]:
321 """
322 Based on class annotations, this prepares the data for `load_into_recurse`.
324 1. convert config-keys to python compatible config_keys
325 2. loads custom class type annotations with the same logic (see also `load_recursive`)
326 3. ensures the annotated types match the actual types after loading the config file.
327 """
328 annotations = all_annotations(cls, _except=_except)
330 to_load = convert_config(data)
331 to_load = load_recursive(cls, to_load, annotations)
332 to_load = ensure_types(to_load, annotations)
333 return to_load
336def load_into_recurse(
337 cls: typing.Type[C],
338 data: dict[str, typing.Any],
339 init: dict[str, typing.Any] = None,
340) -> C:
341 """
342 Loads an instance of `cls` filled with `data`.
344 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
345 `init` can be used to optionally pass extra __init__ arguments. \
346 NOTE: This will overwrite a config key with the same name!
347 """
348 if init is None:
349 init = {}
351 # fixme: cls.__init__ can set other keys than the name is in kwargs!!
353 if is_dataclass(cls):
354 to_load = _check_and_convert_data(cls, data, init.keys())
355 to_load |= init # add extra init variables (should not happen for a dataclass but whatev)
357 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
358 inst = typing.cast(C, cls(**to_load))
359 else:
360 inst = cls(**init)
361 to_load = _check_and_convert_data(cls, data, inst.__dict__.keys())
362 inst.__dict__.update(**to_load)
364 return inst
367def load_into_existing(
368 inst: C,
369 cls: typing.Type[C],
370 data: dict[str, typing.Any],
371 init: dict[str, typing.Any] = None,
372) -> C:
373 """
374 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
375 and thus does not support init.
377 """
378 if init is not None:
379 raise ValueError("Can not init an existing instance!")
381 existing_data = inst.__dict__
383 annotations = all_annotations(cls, _except=existing_data.keys())
384 to_load = convert_config(data)
385 to_load = load_recursive(cls, to_load, annotations)
386 to_load = ensure_types(to_load, annotations)
388 inst.__dict__.update(**to_load)
390 return inst
393def load_into_class(
394 cls: typing.Type[C],
395 data: T_data,
396 /,
397 key: str = None,
398 init: dict[str, typing.Any] = None,
399) -> C:
400 """
401 Shortcut for _load_data + load_into_recurse.
402 """
403 to_load = _load_data(data, key, cls.__name__)
404 return load_into_recurse(cls, to_load, init=init)
407def load_into_instance(
408 inst: C,
409 data: T_data,
410 /,
411 key: str = None,
412 init: dict[str, typing.Any] = None,
413) -> C:
414 """
415 Shortcut for _load_data + load_into_existing.
416 """
417 cls = inst.__class__
418 to_load = _load_data(data, key, cls.__name__)
419 return load_into_existing(inst, cls, to_load, init=init)
422def load_into(
423 cls: typing.Type[C] | C,
424 data: T_data,
425 /,
426 key: str = None,
427 init: dict[str, typing.Any] = None,
428) -> C:
429 """
430 Load your config into a class (instance).
432 Args:
433 cls: either a class or an existing instance of that class.
434 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
435 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
436 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
438 """
439 if not isinstance(cls, type):
440 return load_into_instance(cls, data, key=key, init=init)
442 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
443 _cls = typing.cast(typing.Type[C], cls)
444 return load_into_class(_cls, data, key=key, init=init)