Coverage for src/configuraptor/core.py: 100%
175 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-09-20 10:36 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-09-20 10:36 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import typing
7import warnings
8from pathlib import Path
10from . import loaders
11from .abs import C, T, T_data, Type_C
12from .binary_config import BinaryConfig
13from .errors import (
14 ConfigErrorCouldNotConvert,
15 ConfigErrorInvalidType,
16 ConfigErrorMissingKey,
17)
18from .helpers import (
19 all_annotations,
20 camel_to_snake,
21 check_type,
22 dataclass_field,
23 find_pyproject_toml,
24 is_custom_class,
25 is_optional,
26 is_parameterized,
27)
28from .postpone import Postponed
29from .type_converters import CONVERTERS
32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
33 """
34 If a key contains a dot, traverse the raw dict until the right key was found.
36 Example:
37 key = some.nested.key
38 raw = {"some": {"nested": {"key": {"with": "data"}}}}
39 -> {"with": "data"}
40 """
41 parts = key.split(".")
42 while parts:
43 key = parts.pop(0)
44 if key not in raw:
45 return {}
47 raw = raw[key]
49 return raw
52def _guess_key(clsname: str) -> str:
53 """
54 If no key is manually defined for `load_into`, \
55 the class' name is converted to snake_case to use as the default key.
56 """
57 return camel_to_snake(clsname)
60def __load_data(
61 data: T_data,
62 key: str = None,
63 classname: str = None,
64 lower_keys: bool = False,
65 allow_types: tuple[type, ...] = (dict,),
66) -> dict[str, typing.Any]:
67 """
68 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
70 E.g. class Tool will be mapped to key tool.
71 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
72 """
73 if isinstance(data, bytes):
74 # instantly return, don't modify
75 # bytes as inputs -> bytes as output
76 # but since `T_data` is re-used, that's kind of hard to type for mypy.
77 return data # type: ignore
79 if isinstance(data, list):
80 if not data:
81 raise ValueError("Empty list passed!")
83 final_data: dict[str, typing.Any] = {}
84 for source in data:
85 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types)
87 return final_data
89 if isinstance(data, str):
90 data = Path(data)
92 if isinstance(data, Path):
93 with data.open("rb") as f:
94 loader = loaders.get(data.suffix or data.name)
95 data = loader(f, data.resolve())
97 if not data:
98 return {}
100 if key is None:
101 # try to guess key by grabbing the first one or using the class name
102 if len(data) == 1:
103 key = next(iter(data.keys()))
104 elif classname is not None:
105 key = _guess_key(classname)
107 if key:
108 data = _data_for_nested_key(key, data)
110 if not data:
111 raise ValueError("No data found!")
113 if not isinstance(data, allow_types):
114 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!")
116 if lower_keys and isinstance(data, dict):
117 data = {k.lower(): v for k, v in data.items()}
119 return data
122def _load_data(
123 data: T_data,
124 key: str = None,
125 classname: str = None,
126 lower_keys: bool = False,
127 allow_types: tuple[type, ...] = (dict,),
128) -> dict[str, typing.Any]:
129 """
130 Wrapper around __load_data that retries with key="" if anything goes wrong.
131 """
132 if data is None:
133 # try to load pyproject.toml
134 data = find_pyproject_toml()
136 try:
137 return __load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types)
138 except Exception as e:
139 if key != "":
140 return __load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types)
141 else: # pragma: no cover
142 warnings.warn(f"Data could not be loaded: {e}", source=e)
143 # key already was "", just return data!
144 # (will probably not happen but fallback)
145 return {}
148F = typing.TypeVar("F")
151def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:
152 """
153 Convert a value between types.
154 """
155 if converter := CONVERTERS.get((from_type, to_type)):
156 return typing.cast(T, converter(from_value))
158 # default: just convert type:
159 return to_type(from_value) # type: ignore
162def ensure_types(
163 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False
164) -> dict[str, T | None]:
165 """
166 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
168 If an annotated key in missing from data, it will be filled with None for convenience.
170 TODO: python 3.11 exception groups to throw multiple errors at once!
171 """
172 # custom object to use instead of None, since typing.Optional can be None!
173 # cast to T to make mypy happy
174 notfound = typing.cast(T, object())
175 postponed = Postponed()
177 final: dict[str, T | None] = {}
178 for key, _type in annotations.items():
179 compare = data.get(key, notfound)
180 if compare is notfound: # pragma: nocover
181 warnings.warn(
182 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
183 )
184 # skip!
185 continue
187 if compare is postponed:
188 # don't do anything with this item!
189 continue
191 if not check_type(compare, _type):
192 if convert_types:
193 try:
194 compare = convert_between(compare, type(compare), _type)
195 except (TypeError, ValueError) as e:
196 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e
197 else:
198 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
200 final[key] = compare
202 return final
205def convert_config(items: dict[str, T]) -> dict[str, T]:
206 """
207 Converts the config dict (from toml) or 'overwrites' dict in two ways.
209 1. removes any items where the value is None, since in that case the default should be used;
210 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
211 """
212 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
215Type = typing.Type[typing.Any]
216T_Type = typing.TypeVar("T_Type", bound=Type)
219def load_recursive(
220 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False
221) -> dict[str, T]:
222 """
223 For all annotations (recursively gathered from parents with `all_annotations`), \
224 try to resolve the tree of annotations.
226 Uses `load_into_recurse`, not itself directly.
228 Example:
229 class First:
230 key: str
232 class Second:
233 other: First
235 # step 1
236 cls = Second
237 data = {"second": {"other": {"key": "anything"}}}
238 annotations: {"other": First}
240 # step 1.5
241 data = {"other": {"key": "anything"}
242 annotations: {"other": First}
244 # step 2
245 cls = First
246 data = {"key": "anything"}
247 annotations: {"key": str}
250 TODO: python 3.11 exception groups to throw multiple errors at once!
251 """
252 updated = {}
254 for _key, _type in annotations.items():
255 if _key in data:
256 value: typing.Any = data[_key] # value can change so define it as any instead of T
257 if is_parameterized(_type):
258 origin = typing.get_origin(_type)
259 arguments = typing.get_args(_type)
260 if origin is list and arguments and is_custom_class(arguments[0]):
261 subtype = arguments[0]
262 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]
264 elif origin is dict and arguments and is_custom_class(arguments[1]):
265 # e.g. dict[str, Point]
266 subkeytype, subvaluetype = arguments
267 # subkey(type) is not a custom class, so don't try to convert it:
268 value = {
269 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)
270 for subkey, subvalue in value.items()
271 }
272 # elif origin is dict:
273 # keep data the same
274 elif origin is typing.Union and arguments:
275 for arg in arguments:
276 if is_custom_class(arg):
277 value = _load_into_recurse(arg, value, convert_types=convert_types)
278 else:
279 # print(_type, arg, value)
280 ...
282 # todo: other parameterized/unions/typing.Optional
284 elif is_custom_class(_type):
285 # type must be C (custom class) at this point
286 value = _load_into_recurse(
287 # make mypy and pycharm happy by telling it _type is of type C...
288 # actually just passing _type as first arg!
289 typing.cast(Type_C[typing.Any], _type),
290 value,
291 convert_types=convert_types,
292 )
294 elif _key in cls.__dict__:
295 # property has default, use that instead.
296 value = cls.__dict__[_key]
297 elif is_optional(_type):
298 # type is optional and not found in __dict__ -> default is None
299 value = None
300 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
301 # could have a default factory
302 # todo: do something with field.default?
303 value = field.default_factory()
304 else:
305 raise ConfigErrorMissingKey(_key, cls, _type)
307 updated[_key] = value
309 return updated
312def check_and_convert_data(
313 cls: typing.Type[C],
314 data: dict[str, typing.Any],
315 _except: typing.Iterable[str],
316 strict: bool = True,
317 convert_types: bool = False,
318) -> dict[str, typing.Any]:
319 """
320 Based on class annotations, this prepares the data for `load_into_recurse`.
322 1. convert config-keys to python compatible config_keys
323 2. loads custom class type annotations with the same logic (see also `load_recursive`)
324 3. ensures the annotated types match the actual types after loading the config file.
325 """
326 annotations = all_annotations(cls, _except=_except)
328 to_load = convert_config(data)
329 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)
330 if strict:
331 to_load = ensure_types(to_load, annotations, convert_types=convert_types)
333 return to_load
336T_init_list = list[typing.Any]
337T_init_dict = dict[str, typing.Any]
338T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None
341@typing.no_type_check # (mypy doesn't understand 'match' fully yet)
342def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:
343 """
344 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.
345 """
346 if not init:
347 return [], {}
349 args: T_init_list = []
350 kwargs: T_init_dict = {}
351 match init:
352 case (args, kwargs):
353 return args, kwargs
354 case [*args]:
355 return args, {}
356 case {**kwargs}:
357 return [], kwargs
358 case _:
359 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")
362def _load_into_recurse(
363 cls: typing.Type[C],
364 data: dict[str, typing.Any] | bytes,
365 init: T_init = None,
366 strict: bool = True,
367 convert_types: bool = False,
368) -> C:
369 """
370 Loads an instance of `cls` filled with `data`.
372 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
373 `init` can be used to optionally pass extra __init__ arguments. \
374 NOTE: This will overwrite a config key with the same name!
375 """
376 init_args, init_kwargs = _split_init(init)
378 if isinstance(data, bytes) or issubclass(cls, BinaryConfig):
379 if not isinstance(data, (bytes, dict)): # pragma: no cover
380 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")
381 elif not issubclass(cls, BinaryConfig): # pragma: no cover
382 raise NotImplementedError("Only BinaryConfig can be used with `bytes` (or a dict of bytes) as input.")
384 inst = typing.cast(C, cls._parse_into(data))
385 elif dc.is_dataclass(cls):
386 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)
387 if init:
388 raise ValueError("Init is not allowed for dataclasses!")
390 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
391 inst = typing.cast(C, cls(**to_load))
392 else:
393 inst = cls(*init_args, **init_kwargs)
394 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)
395 inst.__dict__.update(**to_load)
397 return inst
400def _load_into_instance(
401 inst: C,
402 cls: typing.Type[C],
403 data: dict[str, typing.Any],
404 init: T_init = None,
405 strict: bool = True,
406 convert_types: bool = False,
407) -> C:
408 """
409 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
410 and thus does not support init.
412 """
413 if init is not None:
414 raise ValueError("Can not init an existing instance!")
416 existing_data = inst.__dict__
418 to_load = check_and_convert_data(
419 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types
420 )
422 inst.__dict__.update(**to_load)
424 return inst
427def load_into_class(
428 cls: typing.Type[C],
429 data: T_data,
430 /,
431 key: str = None,
432 init: T_init = None,
433 strict: bool = True,
434 lower_keys: bool = False,
435 convert_types: bool = False,
436) -> C:
437 """
438 Shortcut for _load_data + load_into_recurse.
439 """
440 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)
441 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)
442 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)
445def load_into_instance(
446 inst: C,
447 data: T_data,
448 /,
449 key: str = None,
450 init: T_init = None,
451 strict: bool = True,
452 lower_keys: bool = False,
453 convert_types: bool = False,
454) -> C:
455 """
456 Shortcut for _load_data + load_into_existing.
457 """
458 cls = inst.__class__
459 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)
460 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)
461 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)
464def load_into(
465 cls: typing.Type[C],
466 data: T_data = None,
467 /,
468 key: str = None,
469 init: T_init = None,
470 strict: bool = True,
471 lower_keys: bool = False,
472 convert_types: bool = False,
473) -> C:
474 """
475 Load your config into a class (instance).
477 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
478 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
480 Args:
481 cls: either a class or an existing instance of that class.
482 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
483 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
484 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
485 strict: enable type checks or allow anything?
486 lower_keys: should the config keys be lowercased? (for .env)
487 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)
489 """
490 if not isinstance(cls, type):
491 # would not be supported according to mypy, but you can still load_into(instance)
492 return load_into_instance(
493 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
494 )
496 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
497 # _cls = typing.cast(typing.Type[C], cls)
498 return load_into_class(
499 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
500 )