Coverage for src/configuraptor/core.py: 100%

175 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-09-20 10:36 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import typing 

7import warnings 

8from pathlib import Path 

9 

10from . import loaders 

11from .abs import C, T, T_data, Type_C 

12from .binary_config import BinaryConfig 

13from .errors import ( 

14 ConfigErrorCouldNotConvert, 

15 ConfigErrorInvalidType, 

16 ConfigErrorMissingKey, 

17) 

18from .helpers import ( 

19 all_annotations, 

20 camel_to_snake, 

21 check_type, 

22 dataclass_field, 

23 find_pyproject_toml, 

24 is_custom_class, 

25 is_optional, 

26 is_parameterized, 

27) 

28from .postpone import Postponed 

29from .type_converters import CONVERTERS 

30 

31 

32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

33 """ 

34 If a key contains a dot, traverse the raw dict until the right key was found. 

35 

36 Example: 

37 key = some.nested.key 

38 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

39 -> {"with": "data"} 

40 """ 

41 parts = key.split(".") 

42 while parts: 

43 key = parts.pop(0) 

44 if key not in raw: 

45 return {} 

46 

47 raw = raw[key] 

48 

49 return raw 

50 

51 

52def _guess_key(clsname: str) -> str: 

53 """ 

54 If no key is manually defined for `load_into`, \ 

55 the class' name is converted to snake_case to use as the default key. 

56 """ 

57 return camel_to_snake(clsname) 

58 

59 

60def __load_data( 

61 data: T_data, 

62 key: str = None, 

63 classname: str = None, 

64 lower_keys: bool = False, 

65 allow_types: tuple[type, ...] = (dict,), 

66) -> dict[str, typing.Any]: 

67 """ 

68 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

69 

70 E.g. class Tool will be mapped to key tool. 

71 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

72 """ 

73 if isinstance(data, bytes): 

74 # instantly return, don't modify 

75 # bytes as inputs -> bytes as output 

76 # but since `T_data` is re-used, that's kind of hard to type for mypy. 

77 return data # type: ignore 

78 

79 if isinstance(data, list): 

80 if not data: 

81 raise ValueError("Empty list passed!") 

82 

83 final_data: dict[str, typing.Any] = {} 

84 for source in data: 

85 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types) 

86 

87 return final_data 

88 

89 if isinstance(data, str): 

90 data = Path(data) 

91 

92 if isinstance(data, Path): 

93 with data.open("rb") as f: 

94 loader = loaders.get(data.suffix or data.name) 

95 data = loader(f, data.resolve()) 

96 

97 if not data: 

98 return {} 

99 

100 if key is None: 

101 # try to guess key by grabbing the first one or using the class name 

102 if len(data) == 1: 

103 key = next(iter(data.keys())) 

104 elif classname is not None: 

105 key = _guess_key(classname) 

106 

107 if key: 

108 data = _data_for_nested_key(key, data) 

109 

110 if not data: 

111 raise ValueError("No data found!") 

112 

113 if not isinstance(data, allow_types): 

114 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!") 

115 

116 if lower_keys and isinstance(data, dict): 

117 data = {k.lower(): v for k, v in data.items()} 

118 

119 return data 

120 

121 

122def _load_data( 

123 data: T_data, 

124 key: str = None, 

125 classname: str = None, 

126 lower_keys: bool = False, 

127 allow_types: tuple[type, ...] = (dict,), 

128) -> dict[str, typing.Any]: 

129 """ 

130 Wrapper around __load_data that retries with key="" if anything goes wrong. 

131 """ 

132 if data is None: 

133 # try to load pyproject.toml 

134 data = find_pyproject_toml() 

135 

136 try: 

137 return __load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types) 

138 except Exception as e: 

139 if key != "": 

140 return __load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types) 

141 else: # pragma: no cover 

142 warnings.warn(f"Data could not be loaded: {e}", source=e) 

143 # key already was "", just return data! 

144 # (will probably not happen but fallback) 

145 return {} 

146 

147 

148F = typing.TypeVar("F") 

149 

150 

151def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T: 

152 """ 

153 Convert a value between types. 

154 """ 

155 if converter := CONVERTERS.get((from_type, to_type)): 

156 return typing.cast(T, converter(from_value)) 

157 

158 # default: just convert type: 

159 return to_type(from_value) # type: ignore 

160 

161 

162def ensure_types( 

163 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False 

164) -> dict[str, T | None]: 

165 """ 

166 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

167 

168 If an annotated key in missing from data, it will be filled with None for convenience. 

169 

170 TODO: python 3.11 exception groups to throw multiple errors at once! 

171 """ 

172 # custom object to use instead of None, since typing.Optional can be None! 

173 # cast to T to make mypy happy 

174 notfound = typing.cast(T, object()) 

175 postponed = Postponed() 

176 

177 final: dict[str, T | None] = {} 

178 for key, _type in annotations.items(): 

179 compare = data.get(key, notfound) 

180 if compare is notfound: # pragma: nocover 

181 warnings.warn( 

182 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

183 ) 

184 # skip! 

185 continue 

186 

187 if compare is postponed: 

188 # don't do anything with this item! 

189 continue 

190 

191 if not check_type(compare, _type): 

192 if convert_types: 

193 try: 

194 compare = convert_between(compare, type(compare), _type) 

195 except (TypeError, ValueError) as e: 

196 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e 

197 else: 

198 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

199 

200 final[key] = compare 

201 

202 return final 

203 

204 

205def convert_config(items: dict[str, T]) -> dict[str, T]: 

206 """ 

207 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

208 

209 1. removes any items where the value is None, since in that case the default should be used; 

210 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

211 """ 

212 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

213 

214 

215Type = typing.Type[typing.Any] 

216T_Type = typing.TypeVar("T_Type", bound=Type) 

217 

218 

219def load_recursive( 

220 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False 

221) -> dict[str, T]: 

222 """ 

223 For all annotations (recursively gathered from parents with `all_annotations`), \ 

224 try to resolve the tree of annotations. 

225 

226 Uses `load_into_recurse`, not itself directly. 

227 

228 Example: 

229 class First: 

230 key: str 

231 

232 class Second: 

233 other: First 

234 

235 # step 1 

236 cls = Second 

237 data = {"second": {"other": {"key": "anything"}}} 

238 annotations: {"other": First} 

239 

240 # step 1.5 

241 data = {"other": {"key": "anything"} 

242 annotations: {"other": First} 

243 

244 # step 2 

245 cls = First 

246 data = {"key": "anything"} 

247 annotations: {"key": str} 

248 

249 

250 TODO: python 3.11 exception groups to throw multiple errors at once! 

251 """ 

252 updated = {} 

253 

254 for _key, _type in annotations.items(): 

255 if _key in data: 

256 value: typing.Any = data[_key] # value can change so define it as any instead of T 

257 if is_parameterized(_type): 

258 origin = typing.get_origin(_type) 

259 arguments = typing.get_args(_type) 

260 if origin is list and arguments and is_custom_class(arguments[0]): 

261 subtype = arguments[0] 

262 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value] 

263 

264 elif origin is dict and arguments and is_custom_class(arguments[1]): 

265 # e.g. dict[str, Point] 

266 subkeytype, subvaluetype = arguments 

267 # subkey(type) is not a custom class, so don't try to convert it: 

268 value = { 

269 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types) 

270 for subkey, subvalue in value.items() 

271 } 

272 # elif origin is dict: 

273 # keep data the same 

274 elif origin is typing.Union and arguments: 

275 for arg in arguments: 

276 if is_custom_class(arg): 

277 value = _load_into_recurse(arg, value, convert_types=convert_types) 

278 else: 

279 # print(_type, arg, value) 

280 ... 

281 

282 # todo: other parameterized/unions/typing.Optional 

283 

284 elif is_custom_class(_type): 

285 # type must be C (custom class) at this point 

286 value = _load_into_recurse( 

287 # make mypy and pycharm happy by telling it _type is of type C... 

288 # actually just passing _type as first arg! 

289 typing.cast(Type_C[typing.Any], _type), 

290 value, 

291 convert_types=convert_types, 

292 ) 

293 

294 elif _key in cls.__dict__: 

295 # property has default, use that instead. 

296 value = cls.__dict__[_key] 

297 elif is_optional(_type): 

298 # type is optional and not found in __dict__ -> default is None 

299 value = None 

300 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

301 # could have a default factory 

302 # todo: do something with field.default? 

303 value = field.default_factory() 

304 else: 

305 raise ConfigErrorMissingKey(_key, cls, _type) 

306 

307 updated[_key] = value 

308 

309 return updated 

310 

311 

312def check_and_convert_data( 

313 cls: typing.Type[C], 

314 data: dict[str, typing.Any], 

315 _except: typing.Iterable[str], 

316 strict: bool = True, 

317 convert_types: bool = False, 

318) -> dict[str, typing.Any]: 

319 """ 

320 Based on class annotations, this prepares the data for `load_into_recurse`. 

321 

322 1. convert config-keys to python compatible config_keys 

323 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

324 3. ensures the annotated types match the actual types after loading the config file. 

325 """ 

326 annotations = all_annotations(cls, _except=_except) 

327 

328 to_load = convert_config(data) 

329 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types) 

330 if strict: 

331 to_load = ensure_types(to_load, annotations, convert_types=convert_types) 

332 

333 return to_load 

334 

335 

336T_init_list = list[typing.Any] 

337T_init_dict = dict[str, typing.Any] 

338T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

339 

340 

341@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

342def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

343 """ 

344 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

345 """ 

346 if not init: 

347 return [], {} 

348 

349 args: T_init_list = [] 

350 kwargs: T_init_dict = {} 

351 match init: 

352 case (args, kwargs): 

353 return args, kwargs 

354 case [*args]: 

355 return args, {} 

356 case {**kwargs}: 

357 return [], kwargs 

358 case _: 

359 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

360 

361 

362def _load_into_recurse( 

363 cls: typing.Type[C], 

364 data: dict[str, typing.Any] | bytes, 

365 init: T_init = None, 

366 strict: bool = True, 

367 convert_types: bool = False, 

368) -> C: 

369 """ 

370 Loads an instance of `cls` filled with `data`. 

371 

372 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

373 `init` can be used to optionally pass extra __init__ arguments. \ 

374 NOTE: This will overwrite a config key with the same name! 

375 """ 

376 init_args, init_kwargs = _split_init(init) 

377 

378 if isinstance(data, bytes) or issubclass(cls, BinaryConfig): 

379 if not isinstance(data, (bytes, dict)): # pragma: no cover 

380 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.") 

381 elif not issubclass(cls, BinaryConfig): # pragma: no cover 

382 raise NotImplementedError("Only BinaryConfig can be used with `bytes` (or a dict of bytes) as input.") 

383 

384 inst = typing.cast(C, cls._parse_into(data)) 

385 elif dc.is_dataclass(cls): 

386 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types) 

387 if init: 

388 raise ValueError("Init is not allowed for dataclasses!") 

389 

390 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

391 inst = typing.cast(C, cls(**to_load)) 

392 else: 

393 inst = cls(*init_args, **init_kwargs) 

394 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types) 

395 inst.__dict__.update(**to_load) 

396 

397 return inst 

398 

399 

400def _load_into_instance( 

401 inst: C, 

402 cls: typing.Type[C], 

403 data: dict[str, typing.Any], 

404 init: T_init = None, 

405 strict: bool = True, 

406 convert_types: bool = False, 

407) -> C: 

408 """ 

409 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

410 and thus does not support init. 

411 

412 """ 

413 if init is not None: 

414 raise ValueError("Can not init an existing instance!") 

415 

416 existing_data = inst.__dict__ 

417 

418 to_load = check_and_convert_data( 

419 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types 

420 ) 

421 

422 inst.__dict__.update(**to_load) 

423 

424 return inst 

425 

426 

427def load_into_class( 

428 cls: typing.Type[C], 

429 data: T_data, 

430 /, 

431 key: str = None, 

432 init: T_init = None, 

433 strict: bool = True, 

434 lower_keys: bool = False, 

435 convert_types: bool = False, 

436) -> C: 

437 """ 

438 Shortcut for _load_data + load_into_recurse. 

439 """ 

440 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,) 

441 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types) 

442 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types) 

443 

444 

445def load_into_instance( 

446 inst: C, 

447 data: T_data, 

448 /, 

449 key: str = None, 

450 init: T_init = None, 

451 strict: bool = True, 

452 lower_keys: bool = False, 

453 convert_types: bool = False, 

454) -> C: 

455 """ 

456 Shortcut for _load_data + load_into_existing. 

457 """ 

458 cls = inst.__class__ 

459 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,) 

460 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types) 

461 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types) 

462 

463 

464def load_into( 

465 cls: typing.Type[C], 

466 data: T_data = None, 

467 /, 

468 key: str = None, 

469 init: T_init = None, 

470 strict: bool = True, 

471 lower_keys: bool = False, 

472 convert_types: bool = False, 

473) -> C: 

474 """ 

475 Load your config into a class (instance). 

476 

477 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

478 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

479 

480 Args: 

481 cls: either a class or an existing instance of that class. 

482 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

483 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

484 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

485 strict: enable type checks or allow anything? 

486 lower_keys: should the config keys be lowercased? (for .env) 

487 convert_types: should the types be converted to the annotated type if not yet matching? (for .env) 

488 

489 """ 

490 if not isinstance(cls, type): 

491 # would not be supported according to mypy, but you can still load_into(instance) 

492 return load_into_instance( 

493 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

494 ) 

495 

496 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

497 # _cls = typing.cast(typing.Type[C], cls) 

498 return load_into_class( 

499 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

500 )