Coverage for src/configuraptor/core.py: 100%

173 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-09-18 15:00 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import typing 

7import warnings 

8from pathlib import Path 

9 

10from . import loaders 

11from .abs import C, T, T_data, Type_C 

12from .binary_config import BinaryConfig 

13from .errors import ( 

14 ConfigErrorCouldNotConvert, 

15 ConfigErrorInvalidType, 

16 ConfigErrorMissingKey, 

17) 

18from .helpers import ( 

19 all_annotations, 

20 camel_to_snake, 

21 check_type, 

22 dataclass_field, 

23 find_pyproject_toml, 

24 is_custom_class, 

25 is_optional, 

26 is_parameterized, 

27) 

28from .postpone import Postponed 

29from .type_converters import CONVERTERS 

30 

31 

32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

33 """ 

34 If a key contains a dot, traverse the raw dict until the right key was found. 

35 

36 Example: 

37 key = some.nested.key 

38 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

39 -> {"with": "data"} 

40 """ 

41 parts = key.split(".") 

42 while parts: 

43 key = parts.pop(0) 

44 if key not in raw: 

45 return {} 

46 

47 raw = raw[key] 

48 

49 return raw 

50 

51 

52def _guess_key(clsname: str) -> str: 

53 """ 

54 If no key is manually defined for `load_into`, \ 

55 the class' name is converted to snake_case to use as the default key. 

56 """ 

57 return camel_to_snake(clsname) 

58 

59 

60def __load_data( 

61 data: T_data, 

62 key: str = None, 

63 classname: str = None, 

64 lower_keys: bool = False, 

65 allow_types: tuple[type, ...] = (dict,), 

66) -> dict[str, typing.Any]: 

67 """ 

68 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

69 

70 E.g. class Tool will be mapped to key tool. 

71 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

72 """ 

73 if isinstance(data, list): 

74 if not data: 

75 raise ValueError("Empty list passed!") 

76 

77 final_data: dict[str, typing.Any] = {} 

78 for source in data: 

79 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types) 

80 

81 return final_data 

82 

83 if isinstance(data, str): 

84 data = Path(data) 

85 if isinstance(data, Path): 

86 with data.open("rb") as f: 

87 loader = loaders.get(data.suffix or data.name) 

88 data = loader(f, data.resolve()) 

89 if not data: 

90 return {} 

91 

92 if key is None: 

93 # try to guess key by grabbing the first one or using the class name 

94 if len(data) == 1: 

95 key = next(iter(data.keys())) 

96 elif classname is not None: 

97 key = _guess_key(classname) 

98 

99 if key: 

100 data = _data_for_nested_key(key, data) 

101 

102 if not data: 

103 raise ValueError("No data found!") 

104 

105 if not isinstance(data, allow_types): 

106 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!") 

107 

108 if lower_keys and isinstance(data, dict): 

109 data = {k.lower(): v for k, v in data.items()} 

110 

111 return data 

112 

113 

114def _load_data( 

115 data: T_data, 

116 key: str = None, 

117 classname: str = None, 

118 lower_keys: bool = False, 

119 allow_types: tuple[type, ...] = (dict,), 

120) -> dict[str, typing.Any]: 

121 """ 

122 Wrapper around __load_data that retries with key="" if anything goes wrong. 

123 """ 

124 if data is None: 

125 # try to load pyproject.toml 

126 data = find_pyproject_toml() 

127 

128 try: 

129 return __load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types) 

130 except Exception as e: 

131 if key != "": 

132 return __load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types) 

133 else: # pragma: no cover 

134 warnings.warn(f"Data could not be loaded: {e}", source=e) 

135 # key already was "", just return data! 

136 # (will probably not happen but fallback) 

137 return {} 

138 

139 

140F = typing.TypeVar("F") 

141 

142 

143def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T: 

144 """ 

145 Convert a value between types. 

146 """ 

147 if converter := CONVERTERS.get((from_type, to_type)): 

148 return typing.cast(T, converter(from_value)) 

149 

150 # default: just convert type: 

151 return to_type(from_value) # type: ignore 

152 

153 

154def ensure_types( 

155 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False 

156) -> dict[str, T | None]: 

157 """ 

158 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

159 

160 If an annotated key in missing from data, it will be filled with None for convenience. 

161 

162 TODO: python 3.11 exception groups to throw multiple errors at once! 

163 """ 

164 # custom object to use instead of None, since typing.Optional can be None! 

165 # cast to T to make mypy happy 

166 notfound = typing.cast(T, object()) 

167 postponed = Postponed() 

168 

169 final: dict[str, T | None] = {} 

170 for key, _type in annotations.items(): 

171 compare = data.get(key, notfound) 

172 if compare is notfound: # pragma: nocover 

173 warnings.warn( 

174 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

175 ) 

176 # skip! 

177 continue 

178 

179 if compare is postponed: 

180 # don't do anything with this item! 

181 continue 

182 

183 if not check_type(compare, _type): 

184 if convert_types: 

185 try: 

186 compare = convert_between(compare, type(compare), _type) 

187 except (TypeError, ValueError) as e: 

188 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e 

189 else: 

190 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

191 

192 final[key] = compare 

193 

194 return final 

195 

196 

197def convert_config(items: dict[str, T]) -> dict[str, T]: 

198 """ 

199 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

200 

201 1. removes any items where the value is None, since in that case the default should be used; 

202 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

203 """ 

204 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

205 

206 

207Type = typing.Type[typing.Any] 

208T_Type = typing.TypeVar("T_Type", bound=Type) 

209 

210 

211def load_recursive( 

212 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False 

213) -> dict[str, T]: 

214 """ 

215 For all annotations (recursively gathered from parents with `all_annotations`), \ 

216 try to resolve the tree of annotations. 

217 

218 Uses `load_into_recurse`, not itself directly. 

219 

220 Example: 

221 class First: 

222 key: str 

223 

224 class Second: 

225 other: First 

226 

227 # step 1 

228 cls = Second 

229 data = {"second": {"other": {"key": "anything"}}} 

230 annotations: {"other": First} 

231 

232 # step 1.5 

233 data = {"other": {"key": "anything"} 

234 annotations: {"other": First} 

235 

236 # step 2 

237 cls = First 

238 data = {"key": "anything"} 

239 annotations: {"key": str} 

240 

241 

242 TODO: python 3.11 exception groups to throw multiple errors at once! 

243 """ 

244 updated = {} 

245 

246 for _key, _type in annotations.items(): 

247 if _key in data: 

248 value: typing.Any = data[_key] # value can change so define it as any instead of T 

249 if is_parameterized(_type): 

250 origin = typing.get_origin(_type) 

251 arguments = typing.get_args(_type) 

252 if origin is list and arguments and is_custom_class(arguments[0]): 

253 subtype = arguments[0] 

254 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value] 

255 

256 elif origin is dict and arguments and is_custom_class(arguments[1]): 

257 # e.g. dict[str, Point] 

258 subkeytype, subvaluetype = arguments 

259 # subkey(type) is not a custom class, so don't try to convert it: 

260 value = { 

261 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types) 

262 for subkey, subvalue in value.items() 

263 } 

264 # elif origin is dict: 

265 # keep data the same 

266 elif origin is typing.Union and arguments: 

267 for arg in arguments: 

268 if is_custom_class(arg): 

269 value = _load_into_recurse(arg, value, convert_types=convert_types) 

270 else: 

271 # print(_type, arg, value) 

272 ... 

273 

274 # todo: other parameterized/unions/typing.Optional 

275 

276 elif is_custom_class(_type): 

277 # type must be C (custom class) at this point 

278 value = _load_into_recurse( 

279 # make mypy and pycharm happy by telling it _type is of type C... 

280 # actually just passing _type as first arg! 

281 typing.cast(Type_C[typing.Any], _type), 

282 value, 

283 convert_types=convert_types, 

284 ) 

285 

286 elif _key in cls.__dict__: 

287 # property has default, use that instead. 

288 value = cls.__dict__[_key] 

289 elif is_optional(_type): 

290 # type is optional and not found in __dict__ -> default is None 

291 value = None 

292 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

293 # could have a default factory 

294 # todo: do something with field.default? 

295 value = field.default_factory() 

296 else: 

297 raise ConfigErrorMissingKey(_key, cls, _type) 

298 

299 updated[_key] = value 

300 

301 return updated 

302 

303 

304def check_and_convert_data( 

305 cls: typing.Type[C], 

306 data: dict[str, typing.Any], 

307 _except: typing.Iterable[str], 

308 strict: bool = True, 

309 convert_types: bool = False, 

310) -> dict[str, typing.Any]: 

311 """ 

312 Based on class annotations, this prepares the data for `load_into_recurse`. 

313 

314 1. convert config-keys to python compatible config_keys 

315 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

316 3. ensures the annotated types match the actual types after loading the config file. 

317 """ 

318 annotations = all_annotations(cls, _except=_except) 

319 

320 to_load = convert_config(data) 

321 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types) 

322 if strict: 

323 to_load = ensure_types(to_load, annotations, convert_types=convert_types) 

324 

325 return to_load 

326 

327 

328T_init_list = list[typing.Any] 

329T_init_dict = dict[str, typing.Any] 

330T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

331 

332 

333@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

334def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

335 """ 

336 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

337 """ 

338 if not init: 

339 return [], {} 

340 

341 args: T_init_list = [] 

342 kwargs: T_init_dict = {} 

343 match init: 

344 case (args, kwargs): 

345 return args, kwargs 

346 case [*args]: 

347 return args, {} 

348 case {**kwargs}: 

349 return [], kwargs 

350 case _: 

351 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

352 

353 

354def _load_into_recurse( 

355 cls: typing.Type[C], 

356 data: dict[str, typing.Any], 

357 init: T_init = None, 

358 strict: bool = True, 

359 convert_types: bool = False, 

360) -> C: 

361 """ 

362 Loads an instance of `cls` filled with `data`. 

363 

364 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

365 `init` can be used to optionally pass extra __init__ arguments. \ 

366 NOTE: This will overwrite a config key with the same name! 

367 """ 

368 init_args, init_kwargs = _split_init(init) 

369 

370 if issubclass(cls, BinaryConfig): 

371 if not isinstance(data, (bytes, dict)): # pragma: no cover 

372 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.") 

373 inst = typing.cast(C, cls._parse_into(data)) 

374 elif dc.is_dataclass(cls): 

375 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types) 

376 if init: 

377 raise ValueError("Init is not allowed for dataclasses!") 

378 

379 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

380 inst = typing.cast(C, cls(**to_load)) 

381 else: 

382 inst = cls(*init_args, **init_kwargs) 

383 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types) 

384 inst.__dict__.update(**to_load) 

385 

386 return inst 

387 

388 

389def _load_into_instance( 

390 inst: C, 

391 cls: typing.Type[C], 

392 data: dict[str, typing.Any], 

393 init: T_init = None, 

394 strict: bool = True, 

395 convert_types: bool = False, 

396) -> C: 

397 """ 

398 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

399 and thus does not support init. 

400 

401 """ 

402 if init is not None: 

403 raise ValueError("Can not init an existing instance!") 

404 

405 existing_data = inst.__dict__ 

406 

407 to_load = check_and_convert_data( 

408 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types 

409 ) 

410 

411 inst.__dict__.update(**to_load) 

412 

413 return inst 

414 

415 

416def load_into_class( 

417 cls: typing.Type[C], 

418 data: T_data, 

419 /, 

420 key: str = None, 

421 init: T_init = None, 

422 strict: bool = True, 

423 lower_keys: bool = False, 

424 convert_types: bool = False, 

425) -> C: 

426 """ 

427 Shortcut for _load_data + load_into_recurse. 

428 """ 

429 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,) 

430 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types) 

431 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types) 

432 

433 

434def load_into_instance( 

435 inst: C, 

436 data: T_data, 

437 /, 

438 key: str = None, 

439 init: T_init = None, 

440 strict: bool = True, 

441 lower_keys: bool = False, 

442 convert_types: bool = False, 

443) -> C: 

444 """ 

445 Shortcut for _load_data + load_into_existing. 

446 """ 

447 cls = inst.__class__ 

448 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,) 

449 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types) 

450 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types) 

451 

452 

453def load_into( 

454 cls: typing.Type[C], 

455 data: T_data = None, 

456 /, 

457 key: str = None, 

458 init: T_init = None, 

459 strict: bool = True, 

460 lower_keys: bool = False, 

461 convert_types: bool = False, 

462) -> C: 

463 """ 

464 Load your config into a class (instance). 

465 

466 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

467 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

468 

469 Args: 

470 cls: either a class or an existing instance of that class. 

471 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

472 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

473 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

474 strict: enable type checks or allow anything? 

475 lower_keys: should the config keys be lowercased? (for .env) 

476 convert_types: should the types be converted to the annotated type if not yet matching? (for .env) 

477 

478 """ 

479 if not isinstance(cls, type): 

480 # would not be supported according to mypy, but you can still load_into(instance) 

481 return load_into_instance( 

482 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

483 ) 

484 

485 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

486 # _cls = typing.cast(typing.Type[C], cls) 

487 return load_into_class( 

488 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

489 )