Coverage for src/configuraptor/core.py: 100%
158 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 16:44 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 16:44 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import math
7import types
8import typing
9import warnings
10from collections import ChainMap
11from pathlib import Path
13from typeguard import TypeCheckError
14from typeguard import check_type as _check_type
16from . import loaders
17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey
18from .helpers import camel_to_snake
19from .postpone import Postponed
21# T is a reusable typevar
22T = typing.TypeVar("T")
23# t_typelike is anything that can be type hinted
24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union
25# t_data is anything that can be fed to _load_data
26T_data = str | Path | dict[str, typing.Any]
27# c = a config class instance, can be any (user-defined) class
28C = typing.TypeVar("C")
29# type c is a config class
30Type_C = typing.Type[C]
33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
34 """
35 If a key contains a dot, traverse the raw dict until the right key was found.
37 Example:
38 key = some.nested.key
39 raw = {"some": {"nested": {"key": {"with": "data"}}}}
40 -> {"with": "data"}
41 """
42 parts = key.split(".")
43 while parts:
44 raw = raw[parts.pop(0)]
46 return raw
49def _guess_key(clsname: str) -> str:
50 """
51 If no key is manually defined for `load_into`, \
52 the class' name is converted to snake_case to use as the default key.
53 """
54 return camel_to_snake(clsname)
57def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:
58 """
59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
61 E.g. class Tool will be mapped to key tool.
62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
63 """
64 if isinstance(data, str):
65 data = Path(data)
66 if isinstance(data, Path):
67 with data.open("rb") as f:
68 loader = loaders.get(data.suffix)
69 data = loader(f)
71 if not data:
72 return {}
74 if key is None:
75 # try to guess key by grabbing the first one or using the class name
76 if len(data) == 1:
77 key = list(data.keys())[0]
78 elif classname is not None:
79 key = _guess_key(classname)
81 if key:
82 return _data_for_nested_key(key, data)
83 else:
84 # no key found, just return all data
85 return data
88def check_type(value: typing.Any, expected_type: T_typelike) -> bool:
89 """
90 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).
92 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError
93 """
94 try:
95 _check_type(value, expected_type)
96 return True
97 except TypeCheckError:
98 return False
101def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:
102 """
103 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
105 If an annotated key in missing from data, it will be filled with None for convenience.
106 """
107 # custom object to use instead of None, since typing.Optional can be None!
108 # cast to T to make mypy happy
109 notfound = typing.cast(T, object())
110 postponed = Postponed()
112 final: dict[str, T | None] = {}
113 for key, _type in annotations.items():
114 compare = data.get(key, notfound)
115 if compare is notfound: # pragma: nocover
116 warnings.warn(
117 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
118 )
119 # skip!
120 continue
122 if compare is postponed:
123 # don't do anything with this item!
124 continue
126 if not check_type(compare, _type):
127 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
129 final[key] = compare
131 return final
134def convert_config(items: dict[str, T]) -> dict[str, T]:
135 """
136 Converts the config dict (from toml) or 'overwrites' dict in two ways.
138 1. removes any items where the value is None, since in that case the default should be used;
139 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
140 """
141 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
144Type = typing.Type[typing.Any]
145T_Type = typing.TypeVar("T_Type", bound=Type)
148def is_builtin_type(_type: Type) -> bool:
149 """
150 Returns whether _type is one of the builtin types.
151 """
152 return _type.__module__ in ("__builtin__", "builtins")
155# def is_builtin_class_instance(obj: typing.Any) -> bool:
156# return is_builtin_type(obj.__class__)
159def is_from_types_or_typing(_type: Type) -> bool:
160 """
161 Returns whether _type is one of the stlib typing/types types.
163 e.g. types.UnionType or typing.Union
164 """
165 return _type.__module__ in ("types", "typing")
168def is_from_other_toml_supported_module(_type: Type) -> bool:
169 """
170 Besides builtins, toml also supports 'datetime' and 'math' types, \
171 so this returns whether _type is a type from these stdlib modules.
172 """
173 return _type.__module__ in ("datetime", "math")
176def is_parameterized(_type: Type) -> bool:
177 """
178 Returns whether _type is a parameterized type.
180 Examples:
181 list[str] -> True
182 str -> False
183 """
184 return typing.get_origin(_type) is not None
187def is_custom_class(_type: Type) -> bool:
188 """
189 Tries to guess if _type is a builtin or a custom (user-defined) class.
191 Other logic in this module depends on knowing that.
192 """
193 return (
194 type(_type) is type
195 and not is_builtin_type(_type)
196 and not is_from_other_toml_supported_module(_type)
197 and not is_from_types_or_typing(_type)
198 )
201def instance_of_custom_class(var: typing.Any) -> bool:
202 """
203 Calls `is_custom_class` on an instance of a (possibly custom) class.
204 """
205 return is_custom_class(var.__class__)
208def is_optional(_type: Type | None) -> bool:
209 """
210 Tries to guess if _type could be optional.
212 Examples:
213 None -> True
214 NoneType -> True
215 typing.Union[str, None] -> True
216 str | None -> True
217 list[str | None] -> False
218 list[str] -> False
219 """
220 if _type and is_parameterized(_type) and typing.get_origin(_type) in (dict, list):
221 # e.g. list[str]
222 # will crash issubclass to test it first here
223 return False
224 elif _type is math.nan:
225 return False
227 return (
228 _type is None
229 or types.NoneType in typing.get_args(_type) # union with Nonetype
230 or issubclass(types.NoneType, _type)
231 or issubclass(types.NoneType, type(_type)) # no type # Nonetype
232 )
235def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:
236 """
237 Get Field info for a dataclass cls.
238 """
239 fields = getattr(cls, "__dataclass_fields__", {})
240 return fields.get(key)
243def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:
244 """
245 For all annotations (recursively gathered from parents with `all_annotations`), \
246 try to resolve the tree of annotations.
248 Uses `load_into_recurse`, not itself directly.
250 Example:
251 class First:
252 key: str
254 class Second:
255 other: First
257 # step 1
258 cls = Second
259 data = {"second": {"other": {"key": "anything"}}}
260 annotations: {"other": First}
262 # step 1.5
263 data = {"other": {"key": "anything"}
264 annotations: {"other": First}
266 # step 2
267 cls = First
268 data = {"key": "anything"}
269 annotations: {"key": str}
271 """
272 updated = {}
274 for _key, _type in annotations.items():
275 if _key in data:
276 value: typing.Any = data[_key] # value can change so define it as any instead of T
277 if is_parameterized(_type):
278 origin = typing.get_origin(_type)
279 arguments = typing.get_args(_type)
280 if origin is list and arguments and is_custom_class(arguments[0]):
281 subtype = arguments[0]
282 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]
284 elif origin is dict and arguments and is_custom_class(arguments[1]):
285 # e.g. dict[str, Point]
286 subkeytype, subvaluetype = arguments
287 # subkey(type) is not a custom class, so don't try to convert it:
288 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}
289 # elif origin is dict:
290 # keep data the same
291 elif origin is typing.Union and arguments:
292 for arg in arguments:
293 if is_custom_class(arg):
294 value = _load_into_recurse(arg, value)
295 else:
296 # print(_type, arg, value)
297 ...
299 # todo: other parameterized/unions/typing.Optional
301 elif is_custom_class(_type):
302 # type must be C (custom class) at this point
303 value = _load_into_recurse(
304 # make mypy and pycharm happy by telling it _type is of type C...
305 # actually just passing _type as first arg!
306 typing.cast(Type_C[typing.Any], _type),
307 value,
308 )
310 elif _key in cls.__dict__:
311 # property has default, use that instead.
312 value = cls.__dict__[_key]
313 elif is_optional(_type):
314 # type is optional and not found in __dict__ -> default is None
315 value = None
316 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
317 # could have a default factory
318 # todo: do something with field.default?
319 value = field.default_factory()
320 else:
321 raise ConfigErrorMissingKey(_key, cls, _type)
323 updated[_key] = value
325 return updated
328def _all_annotations(cls: Type) -> ChainMap[str, Type]:
329 """
330 Returns a dictionary-like ChainMap that includes annotations for all \
331 attributes defined in cls or inherited from superclasses.
332 """
333 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
336def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:
337 """
338 Wrapper around `_all_annotations` that filters away any keys in _except.
340 It also flattens the ChainMap to a regular dict.
341 """
342 if _except is None:
343 _except = set()
345 _all = _all_annotations(cls)
346 return {k: v for k, v in _all.items() if k not in _except}
349def check_and_convert_data(
350 cls: typing.Type[C],
351 data: dict[str, typing.Any],
352 _except: typing.Iterable[str],
353 strict: bool = True,
354) -> dict[str, typing.Any]:
355 """
356 Based on class annotations, this prepares the data for `load_into_recurse`.
358 1. convert config-keys to python compatible config_keys
359 2. loads custom class type annotations with the same logic (see also `load_recursive`)
360 3. ensures the annotated types match the actual types after loading the config file.
361 """
362 annotations = all_annotations(cls, _except=_except)
364 to_load = convert_config(data)
365 to_load = load_recursive(cls, to_load, annotations)
366 if strict:
367 to_load = ensure_types(to_load, annotations)
369 return to_load
372def _load_into_recurse(
373 cls: typing.Type[C],
374 data: dict[str, typing.Any],
375 init: dict[str, typing.Any] = None,
376 strict: bool = True,
377) -> C:
378 """
379 Loads an instance of `cls` filled with `data`.
381 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
382 `init` can be used to optionally pass extra __init__ arguments. \
383 NOTE: This will overwrite a config key with the same name!
384 """
385 if init is None:
386 init = {}
388 # fixme: cls.__init__ can set other keys than the name is in kwargs!!
390 if dc.is_dataclass(cls):
391 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict)
392 to_load |= init # add extra init variables (should not happen for a dataclass but whatev)
394 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
395 inst = typing.cast(C, cls(**to_load))
396 else:
397 inst = cls(**init)
398 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict)
399 inst.__dict__.update(**to_load)
401 return inst
404def _load_into_instance(
405 inst: C,
406 cls: typing.Type[C],
407 data: dict[str, typing.Any],
408 init: dict[str, typing.Any] = None,
409 strict: bool = True,
410) -> C:
411 """
412 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
413 and thus does not support init.
415 """
416 if init is not None:
417 raise ValueError("Can not init an existing instance!")
419 existing_data = inst.__dict__
421 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict)
423 inst.__dict__.update(**to_load)
425 return inst
428def load_into_class(
429 cls: typing.Type[C],
430 data: T_data,
431 /,
432 key: str = None,
433 init: dict[str, typing.Any] = None,
434 strict: bool = True,
435) -> C:
436 """
437 Shortcut for _load_data + load_into_recurse.
438 """
439 to_load = _load_data(data, key, cls.__name__)
440 return _load_into_recurse(cls, to_load, init=init, strict=strict)
443def load_into_instance(
444 inst: C,
445 data: T_data,
446 /,
447 key: str = None,
448 init: dict[str, typing.Any] = None,
449 strict: bool = True,
450) -> C:
451 """
452 Shortcut for _load_data + load_into_existing.
453 """
454 cls = inst.__class__
455 to_load = _load_data(data, key, cls.__name__)
456 return _load_into_instance(inst, cls, to_load, init=init, strict=strict)
459def load_into(
460 cls: typing.Type[C],
461 data: T_data,
462 /,
463 key: str = None,
464 init: dict[str, typing.Any] = None,
465 strict: bool = True,
466) -> C:
467 """
468 Load your config into a class (instance).
470 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
471 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
473 Args:
474 cls: either a class or an existing instance of that class.
475 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
476 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
477 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
478 strict: enable type checks or allow anything?
480 """
481 if not isinstance(cls, type):
482 # would not be supported according to mypy, but you can still load_into(instance)
483 return load_into_instance(cls, data, key=key, init=init, strict=strict)
485 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
486 # _cls = typing.cast(typing.Type[C], cls)
487 return load_into_class(cls, data, key=key, init=init, strict=strict)