Coverage for src/configuraptor/core.py: 100%
173 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-09-18 14:24 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-09-18 14:24 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import typing
7import warnings
8from pathlib import Path
10from . import loaders
11from .abs import C, T, T_data, Type_C
12from .binary_config import BinaryConfig
13from .errors import (
14 ConfigErrorCouldNotConvert,
15 ConfigErrorInvalidType,
16 ConfigErrorMissingKey,
17)
18from .helpers import (
19 all_annotations,
20 camel_to_snake,
21 check_type,
22 dataclass_field,
23 find_pyproject_toml,
24 is_custom_class,
25 is_optional,
26 is_parameterized,
27)
28from .postpone import Postponed
29from .type_converters import CONVERTERS
32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
33 """
34 If a key contains a dot, traverse the raw dict until the right key was found.
36 Example:
37 key = some.nested.key
38 raw = {"some": {"nested": {"key": {"with": "data"}}}}
39 -> {"with": "data"}
40 """
41 parts = key.split(".")
42 while parts:
43 key = parts.pop(0)
44 if key not in raw:
45 return {}
47 raw = raw[key]
49 return raw
52def _guess_key(clsname: str) -> str:
53 """
54 If no key is manually defined for `load_into`, \
55 the class' name is converted to snake_case to use as the default key.
56 """
57 return camel_to_snake(clsname)
60def __load_data(
61 data: T_data,
62 key: str = None,
63 classname: str = None,
64 lower_keys: bool = False,
65 allow_types: tuple[type, ...] = (dict,),
66) -> dict[str, typing.Any]:
67 """
68 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
70 E.g. class Tool will be mapped to key tool.
71 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
72 """
73 if isinstance(data, list):
74 if not data:
75 raise ValueError("Empty list passed!")
77 final_data: dict[str, typing.Any] = {}
78 for source in data:
79 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types)
81 return final_data
83 if isinstance(data, str):
84 data = Path(data)
85 if isinstance(data, Path):
86 with data.open("rb") as f:
87 loader = loaders.get(data.suffix or data.name)
88 data = loader(f, data.resolve())
89 if not data:
90 return {}
92 if key is None:
93 # try to guess key by grabbing the first one or using the class name
94 if len(data) == 1:
95 key = next(iter(data.keys()))
96 elif classname is not None:
97 key = _guess_key(classname)
99 if key:
100 data = _data_for_nested_key(key, data)
102 if not data:
103 raise ValueError("No data found!")
105 if not isinstance(data, allow_types):
106 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!")
108 if lower_keys and isinstance(data, dict):
109 data = {k.lower(): v for k, v in data.items()}
111 return data
114def _load_data(
115 data: T_data,
116 key: str = None,
117 classname: str = None,
118 lower_keys: bool = False,
119 allow_types: tuple[type, ...] = (dict,),
120) -> dict[str, typing.Any]:
121 """
122 Wrapper around __load_data that retries with key="" if anything goes wrong.
123 """
124 if data is None:
125 # try to load pyproject.toml
126 data = find_pyproject_toml()
128 try:
129 return __load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types)
130 except Exception as e:
131 if key != "":
132 return __load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types)
133 else: # pragma: no cover
134 warnings.warn(f"Data could not be loaded: {e}", source=e)
135 # key already was "", just return data!
136 # (will probably not happen but fallback)
137 return {}
140F = typing.TypeVar("F")
143def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:
144 """
145 Convert a value between types.
146 """
147 if converter := CONVERTERS.get((from_type, to_type)):
148 return typing.cast(T, converter(from_value))
150 # default: just convert type:
151 return to_type(from_value) # type: ignore
154def ensure_types(
155 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False
156) -> dict[str, T | None]:
157 """
158 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
160 If an annotated key in missing from data, it will be filled with None for convenience.
162 TODO: python 3.11 exception groups to throw multiple errors at once!
163 """
164 # custom object to use instead of None, since typing.Optional can be None!
165 # cast to T to make mypy happy
166 notfound = typing.cast(T, object())
167 postponed = Postponed()
169 final: dict[str, T | None] = {}
170 for key, _type in annotations.items():
171 compare = data.get(key, notfound)
172 if compare is notfound: # pragma: nocover
173 warnings.warn(
174 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
175 )
176 # skip!
177 continue
179 if compare is postponed:
180 # don't do anything with this item!
181 continue
183 if not check_type(compare, _type):
184 if convert_types:
185 try:
186 compare = convert_between(compare, type(compare), _type)
187 except (TypeError, ValueError) as e:
188 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e
189 else:
190 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
192 final[key] = compare
194 return final
197def convert_config(items: dict[str, T]) -> dict[str, T]:
198 """
199 Converts the config dict (from toml) or 'overwrites' dict in two ways.
201 1. removes any items where the value is None, since in that case the default should be used;
202 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
203 """
204 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
207Type = typing.Type[typing.Any]
208T_Type = typing.TypeVar("T_Type", bound=Type)
211def load_recursive(
212 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False
213) -> dict[str, T]:
214 """
215 For all annotations (recursively gathered from parents with `all_annotations`), \
216 try to resolve the tree of annotations.
218 Uses `load_into_recurse`, not itself directly.
220 Example:
221 class First:
222 key: str
224 class Second:
225 other: First
227 # step 1
228 cls = Second
229 data = {"second": {"other": {"key": "anything"}}}
230 annotations: {"other": First}
232 # step 1.5
233 data = {"other": {"key": "anything"}
234 annotations: {"other": First}
236 # step 2
237 cls = First
238 data = {"key": "anything"}
239 annotations: {"key": str}
242 TODO: python 3.11 exception groups to throw multiple errors at once!
243 """
244 updated = {}
246 for _key, _type in annotations.items():
247 if _key in data:
248 value: typing.Any = data[_key] # value can change so define it as any instead of T
249 if is_parameterized(_type):
250 origin = typing.get_origin(_type)
251 arguments = typing.get_args(_type)
252 if origin is list and arguments and is_custom_class(arguments[0]):
253 subtype = arguments[0]
254 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]
256 elif origin is dict and arguments and is_custom_class(arguments[1]):
257 # e.g. dict[str, Point]
258 subkeytype, subvaluetype = arguments
259 # subkey(type) is not a custom class, so don't try to convert it:
260 value = {
261 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)
262 for subkey, subvalue in value.items()
263 }
264 # elif origin is dict:
265 # keep data the same
266 elif origin is typing.Union and arguments:
267 for arg in arguments:
268 if is_custom_class(arg):
269 value = _load_into_recurse(arg, value, convert_types=convert_types)
270 else:
271 # print(_type, arg, value)
272 ...
274 # todo: other parameterized/unions/typing.Optional
276 elif is_custom_class(_type):
277 # type must be C (custom class) at this point
278 value = _load_into_recurse(
279 # make mypy and pycharm happy by telling it _type is of type C...
280 # actually just passing _type as first arg!
281 typing.cast(Type_C[typing.Any], _type),
282 value,
283 convert_types=convert_types,
284 )
286 elif _key in cls.__dict__:
287 # property has default, use that instead.
288 value = cls.__dict__[_key]
289 elif is_optional(_type):
290 # type is optional and not found in __dict__ -> default is None
291 value = None
292 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
293 # could have a default factory
294 # todo: do something with field.default?
295 value = field.default_factory()
296 else:
297 raise ConfigErrorMissingKey(_key, cls, _type)
299 updated[_key] = value
301 return updated
304def check_and_convert_data(
305 cls: typing.Type[C],
306 data: dict[str, typing.Any],
307 _except: typing.Iterable[str],
308 strict: bool = True,
309 convert_types: bool = False,
310) -> dict[str, typing.Any]:
311 """
312 Based on class annotations, this prepares the data for `load_into_recurse`.
314 1. convert config-keys to python compatible config_keys
315 2. loads custom class type annotations with the same logic (see also `load_recursive`)
316 3. ensures the annotated types match the actual types after loading the config file.
317 """
318 annotations = all_annotations(cls, _except=_except)
320 to_load = convert_config(data)
321 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)
322 if strict:
323 to_load = ensure_types(to_load, annotations, convert_types=convert_types)
325 return to_load
328T_init_list = list[typing.Any]
329T_init_dict = dict[str, typing.Any]
330T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None
333@typing.no_type_check # (mypy doesn't understand 'match' fully yet)
334def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:
335 """
336 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.
337 """
338 if not init:
339 return [], {}
341 args: T_init_list = []
342 kwargs: T_init_dict = {}
343 match init:
344 case (args, kwargs):
345 return args, kwargs
346 case [*args]:
347 return args, {}
348 case {**kwargs}:
349 return [], kwargs
350 case _:
351 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")
354def _load_into_recurse(
355 cls: typing.Type[C],
356 data: dict[str, typing.Any],
357 init: T_init = None,
358 strict: bool = True,
359 convert_types: bool = False,
360) -> C:
361 """
362 Loads an instance of `cls` filled with `data`.
364 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
365 `init` can be used to optionally pass extra __init__ arguments. \
366 NOTE: This will overwrite a config key with the same name!
367 """
368 init_args, init_kwargs = _split_init(init)
370 if issubclass(cls, BinaryConfig):
371 if not isinstance(data, (bytes, dict)): # pragma: no cover
372 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")
373 inst = typing.cast(C, cls._parse_into(data))
374 elif dc.is_dataclass(cls):
375 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)
376 if init:
377 raise ValueError("Init is not allowed for dataclasses!")
379 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
380 inst = typing.cast(C, cls(**to_load))
381 else:
382 inst = cls(*init_args, **init_kwargs)
383 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)
384 inst.__dict__.update(**to_load)
386 return inst
389def _load_into_instance(
390 inst: C,
391 cls: typing.Type[C],
392 data: dict[str, typing.Any],
393 init: T_init = None,
394 strict: bool = True,
395 convert_types: bool = False,
396) -> C:
397 """
398 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
399 and thus does not support init.
401 """
402 if init is not None:
403 raise ValueError("Can not init an existing instance!")
405 existing_data = inst.__dict__
407 to_load = check_and_convert_data(
408 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types
409 )
411 inst.__dict__.update(**to_load)
413 return inst
416def load_into_class(
417 cls: typing.Type[C],
418 data: T_data,
419 /,
420 key: str = None,
421 init: T_init = None,
422 strict: bool = True,
423 lower_keys: bool = False,
424 convert_types: bool = False,
425) -> C:
426 """
427 Shortcut for _load_data + load_into_recurse.
428 """
429 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)
430 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)
431 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)
434def load_into_instance(
435 inst: C,
436 data: T_data,
437 /,
438 key: str = None,
439 init: T_init = None,
440 strict: bool = True,
441 lower_keys: bool = False,
442 convert_types: bool = False,
443) -> C:
444 """
445 Shortcut for _load_data + load_into_existing.
446 """
447 cls = inst.__class__
448 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)
449 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)
450 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)
453def load_into(
454 cls: typing.Type[C],
455 data: T_data = None,
456 /,
457 key: str = None,
458 init: T_init = None,
459 strict: bool = True,
460 lower_keys: bool = False,
461 convert_types: bool = False,
462) -> C:
463 """
464 Load your config into a class (instance).
466 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
467 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
469 Args:
470 cls: either a class or an existing instance of that class.
471 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
472 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
473 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
474 strict: enable type checks or allow anything?
475 lower_keys: should the config keys be lowercased? (for .env)
476 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)
478 """
479 if not isinstance(cls, type):
480 # would not be supported according to mypy, but you can still load_into(instance)
481 return load_into_instance(
482 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
483 )
485 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
486 # _cls = typing.cast(typing.Type[C], cls)
487 return load_into_class(
488 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
489 )