Coverage for src/configuraptor/core.py: 100%
171 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-09-18 12:35 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-09-18 12:35 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import typing
7import warnings
8from pathlib import Path
10from . import loaders
11from .abs import C, T, T_data, Type_C
12from .binary_config import BinaryConfig
13from .errors import (
14 ConfigErrorCouldNotConvert,
15 ConfigErrorInvalidType,
16 ConfigErrorMissingKey,
17)
18from .helpers import (
19 all_annotations,
20 camel_to_snake,
21 check_type,
22 dataclass_field,
23 find_pyproject_toml,
24 is_custom_class,
25 is_optional,
26 is_parameterized,
27)
28from .postpone import Postponed
29from .type_converters import CONVERTERS
32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
33 """
34 If a key contains a dot, traverse the raw dict until the right key was found.
36 Example:
37 key = some.nested.key
38 raw = {"some": {"nested": {"key": {"with": "data"}}}}
39 -> {"with": "data"}
40 """
41 parts = key.split(".")
42 while parts:
43 key = parts.pop(0)
44 if key not in raw:
45 return {}
47 raw = raw[key]
49 return raw
52def _guess_key(clsname: str) -> str:
53 """
54 If no key is manually defined for `load_into`, \
55 the class' name is converted to snake_case to use as the default key.
56 """
57 return camel_to_snake(clsname)
60def __load_data(
61 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False
62) -> dict[str, typing.Any]:
63 """
64 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
66 E.g. class Tool will be mapped to key tool.
67 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
68 """
69 if isinstance(data, list):
70 if not data:
71 raise ValueError("Empty list passed!")
73 final_data: dict[str, typing.Any] = {}
74 for source in data:
75 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True)
77 return final_data
79 if isinstance(data, str):
80 data = Path(data)
81 if isinstance(data, Path):
82 with data.open("rb") as f:
83 loader = loaders.get(data.suffix or data.name)
84 data = loader(f, data.resolve())
85 if not data:
86 return {}
88 if key is None:
89 # try to guess key by grabbing the first one or using the class name
90 if len(data) == 1:
91 key = next(iter(data.keys()))
92 elif classname is not None:
93 key = _guess_key(classname)
95 if key:
96 data = _data_for_nested_key(key, data)
98 if not data:
99 raise ValueError("No data found!")
101 if not isinstance(data, dict):
102 raise ValueError("Data is not a dict!")
104 if lower_keys:
105 data = {k.lower(): v for k, v in data.items()}
107 return data
110def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]:
111 """
112 Wrapper around __load_data that retries with key="" if anything goes wrong.
113 """
114 if data is None:
115 # try to load pyproject.toml
116 data = find_pyproject_toml()
118 try:
119 return __load_data(data, key, classname, lower_keys=lower_keys)
120 except Exception as e:
121 if key != "":
122 return __load_data(data, "", classname, lower_keys=lower_keys)
123 else: # pragma: no cover
124 warnings.warn(f"Data could not be loaded: {e}", source=e)
125 # key already was "", just return data!
126 # (will probably not happen but fallback)
127 return {}
130F = typing.TypeVar("F")
133def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:
134 """
135 Convert a value between types.
136 """
137 if converter := CONVERTERS.get((from_type, to_type)):
138 return typing.cast(T, converter(from_value))
140 # default: just convert type:
141 return to_type(from_value) # type: ignore
144def ensure_types(
145 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False
146) -> dict[str, T | None]:
147 """
148 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
150 If an annotated key in missing from data, it will be filled with None for convenience.
152 TODO: python 3.11 exception groups to throw multiple errors at once!
153 """
154 # custom object to use instead of None, since typing.Optional can be None!
155 # cast to T to make mypy happy
156 notfound = typing.cast(T, object())
157 postponed = Postponed()
159 final: dict[str, T | None] = {}
160 for key, _type in annotations.items():
161 compare = data.get(key, notfound)
162 if compare is notfound: # pragma: nocover
163 warnings.warn(
164 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
165 )
166 # skip!
167 continue
169 if compare is postponed:
170 # don't do anything with this item!
171 continue
173 if not check_type(compare, _type):
174 if convert_types:
175 try:
176 compare = convert_between(compare, type(compare), _type)
177 except (TypeError, ValueError) as e:
178 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e
179 else:
180 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
182 final[key] = compare
184 return final
187def convert_config(items: dict[str, T]) -> dict[str, T]:
188 """
189 Converts the config dict (from toml) or 'overwrites' dict in two ways.
191 1. removes any items where the value is None, since in that case the default should be used;
192 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
193 """
194 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
197Type = typing.Type[typing.Any]
198T_Type = typing.TypeVar("T_Type", bound=Type)
201def load_recursive(
202 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False
203) -> dict[str, T]:
204 """
205 For all annotations (recursively gathered from parents with `all_annotations`), \
206 try to resolve the tree of annotations.
208 Uses `load_into_recurse`, not itself directly.
210 Example:
211 class First:
212 key: str
214 class Second:
215 other: First
217 # step 1
218 cls = Second
219 data = {"second": {"other": {"key": "anything"}}}
220 annotations: {"other": First}
222 # step 1.5
223 data = {"other": {"key": "anything"}
224 annotations: {"other": First}
226 # step 2
227 cls = First
228 data = {"key": "anything"}
229 annotations: {"key": str}
232 TODO: python 3.11 exception groups to throw multiple errors at once!
233 """
234 updated = {}
236 for _key, _type in annotations.items():
237 if _key in data:
238 value: typing.Any = data[_key] # value can change so define it as any instead of T
239 if is_parameterized(_type):
240 origin = typing.get_origin(_type)
241 arguments = typing.get_args(_type)
242 if origin is list and arguments and is_custom_class(arguments[0]):
243 subtype = arguments[0]
244 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]
246 elif origin is dict and arguments and is_custom_class(arguments[1]):
247 # e.g. dict[str, Point]
248 subkeytype, subvaluetype = arguments
249 # subkey(type) is not a custom class, so don't try to convert it:
250 value = {
251 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)
252 for subkey, subvalue in value.items()
253 }
254 # elif origin is dict:
255 # keep data the same
256 elif origin is typing.Union and arguments:
257 for arg in arguments:
258 if is_custom_class(arg):
259 value = _load_into_recurse(arg, value, convert_types=convert_types)
260 else:
261 # print(_type, arg, value)
262 ...
264 # todo: other parameterized/unions/typing.Optional
266 elif is_custom_class(_type):
267 # type must be C (custom class) at this point
268 value = _load_into_recurse(
269 # make mypy and pycharm happy by telling it _type is of type C...
270 # actually just passing _type as first arg!
271 typing.cast(Type_C[typing.Any], _type),
272 value,
273 convert_types=convert_types,
274 )
276 elif _key in cls.__dict__:
277 # property has default, use that instead.
278 value = cls.__dict__[_key]
279 elif is_optional(_type):
280 # type is optional and not found in __dict__ -> default is None
281 value = None
282 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
283 # could have a default factory
284 # todo: do something with field.default?
285 value = field.default_factory()
286 else:
287 raise ConfigErrorMissingKey(_key, cls, _type)
289 updated[_key] = value
291 return updated
294def check_and_convert_data(
295 cls: typing.Type[C],
296 data: dict[str, typing.Any],
297 _except: typing.Iterable[str],
298 strict: bool = True,
299 convert_types: bool = False,
300) -> dict[str, typing.Any]:
301 """
302 Based on class annotations, this prepares the data for `load_into_recurse`.
304 1. convert config-keys to python compatible config_keys
305 2. loads custom class type annotations with the same logic (see also `load_recursive`)
306 3. ensures the annotated types match the actual types after loading the config file.
307 """
308 annotations = all_annotations(cls, _except=_except)
310 to_load = convert_config(data)
311 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)
312 if strict:
313 to_load = ensure_types(to_load, annotations, convert_types=convert_types)
315 return to_load
318T_init_list = list[typing.Any]
319T_init_dict = dict[str, typing.Any]
320T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None
323@typing.no_type_check # (mypy doesn't understand 'match' fully yet)
324def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:
325 """
326 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.
327 """
328 if not init:
329 return [], {}
331 args: T_init_list = []
332 kwargs: T_init_dict = {}
333 match init:
334 case (args, kwargs):
335 return args, kwargs
336 case [*args]:
337 return args, {}
338 case {**kwargs}:
339 return [], kwargs
340 case _:
341 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")
344def _load_into_recurse(
345 cls: typing.Type[C],
346 data: dict[str, typing.Any],
347 init: T_init = None,
348 strict: bool = True,
349 convert_types: bool = False,
350) -> C:
351 """
352 Loads an instance of `cls` filled with `data`.
354 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
355 `init` can be used to optionally pass extra __init__ arguments. \
356 NOTE: This will overwrite a config key with the same name!
357 """
358 init_args, init_kwargs = _split_init(init)
360 if issubclass(cls, BinaryConfig):
361 # todo: init?
363 if not isinstance(data, (bytes, dict)): # pragma: no cover
364 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")
365 inst = typing.cast(C, cls._parse_into(data))
366 elif dc.is_dataclass(cls):
367 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)
368 if init:
369 raise ValueError("Init is not allowed for dataclasses!")
371 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
372 inst = typing.cast(C, cls(**to_load))
373 else:
374 inst = cls(*init_args, **init_kwargs)
375 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)
376 inst.__dict__.update(**to_load)
378 return inst
381def _load_into_instance(
382 inst: C,
383 cls: typing.Type[C],
384 data: dict[str, typing.Any],
385 init: T_init = None,
386 strict: bool = True,
387 convert_types: bool = False,
388) -> C:
389 """
390 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
391 and thus does not support init.
393 """
394 if init is not None:
395 raise ValueError("Can not init an existing instance!")
397 existing_data = inst.__dict__
399 to_load = check_and_convert_data(
400 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types
401 )
403 inst.__dict__.update(**to_load)
405 return inst
408def load_into_class(
409 cls: typing.Type[C],
410 data: T_data,
411 /,
412 key: str = None,
413 init: T_init = None,
414 strict: bool = True,
415 lower_keys: bool = False,
416 convert_types: bool = False,
417) -> C:
418 """
419 Shortcut for _load_data + load_into_recurse.
420 """
421 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)
422 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)
425def load_into_instance(
426 inst: C,
427 data: T_data,
428 /,
429 key: str = None,
430 init: T_init = None,
431 strict: bool = True,
432 lower_keys: bool = False,
433 convert_types: bool = False,
434) -> C:
435 """
436 Shortcut for _load_data + load_into_existing.
437 """
438 cls = inst.__class__
439 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)
440 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)
443def load_into(
444 cls: typing.Type[C],
445 data: T_data = None,
446 /,
447 key: str = None,
448 init: T_init = None,
449 strict: bool = True,
450 lower_keys: bool = False,
451 convert_types: bool = False,
452) -> C:
453 """
454 Load your config into a class (instance).
456 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
457 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
459 Args:
460 cls: either a class or an existing instance of that class.
461 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
462 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
463 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
464 strict: enable type checks or allow anything?
465 lower_keys: should the config keys be lowercased? (for .env)
466 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)
468 """
469 if not isinstance(cls, type):
470 # would not be supported according to mypy, but you can still load_into(instance)
471 return load_into_instance(
472 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
473 )
475 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
476 # _cls = typing.cast(typing.Type[C], cls)
477 return load_into_class(
478 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types
479 )