Coverage for src/configuraptor/core.py: 100%

171 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-09-18 12:35 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import typing 

7import warnings 

8from pathlib import Path 

9 

10from . import loaders 

11from .abs import C, T, T_data, Type_C 

12from .binary_config import BinaryConfig 

13from .errors import ( 

14 ConfigErrorCouldNotConvert, 

15 ConfigErrorInvalidType, 

16 ConfigErrorMissingKey, 

17) 

18from .helpers import ( 

19 all_annotations, 

20 camel_to_snake, 

21 check_type, 

22 dataclass_field, 

23 find_pyproject_toml, 

24 is_custom_class, 

25 is_optional, 

26 is_parameterized, 

27) 

28from .postpone import Postponed 

29from .type_converters import CONVERTERS 

30 

31 

32def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

33 """ 

34 If a key contains a dot, traverse the raw dict until the right key was found. 

35 

36 Example: 

37 key = some.nested.key 

38 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

39 -> {"with": "data"} 

40 """ 

41 parts = key.split(".") 

42 while parts: 

43 key = parts.pop(0) 

44 if key not in raw: 

45 return {} 

46 

47 raw = raw[key] 

48 

49 return raw 

50 

51 

52def _guess_key(clsname: str) -> str: 

53 """ 

54 If no key is manually defined for `load_into`, \ 

55 the class' name is converted to snake_case to use as the default key. 

56 """ 

57 return camel_to_snake(clsname) 

58 

59 

60def __load_data( 

61 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False 

62) -> dict[str, typing.Any]: 

63 """ 

64 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

65 

66 E.g. class Tool will be mapped to key tool. 

67 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

68 """ 

69 if isinstance(data, list): 

70 if not data: 

71 raise ValueError("Empty list passed!") 

72 

73 final_data: dict[str, typing.Any] = {} 

74 for source in data: 

75 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True) 

76 

77 return final_data 

78 

79 if isinstance(data, str): 

80 data = Path(data) 

81 if isinstance(data, Path): 

82 with data.open("rb") as f: 

83 loader = loaders.get(data.suffix or data.name) 

84 data = loader(f, data.resolve()) 

85 if not data: 

86 return {} 

87 

88 if key is None: 

89 # try to guess key by grabbing the first one or using the class name 

90 if len(data) == 1: 

91 key = next(iter(data.keys())) 

92 elif classname is not None: 

93 key = _guess_key(classname) 

94 

95 if key: 

96 data = _data_for_nested_key(key, data) 

97 

98 if not data: 

99 raise ValueError("No data found!") 

100 

101 if not isinstance(data, dict): 

102 raise ValueError("Data is not a dict!") 

103 

104 if lower_keys: 

105 data = {k.lower(): v for k, v in data.items()} 

106 

107 return data 

108 

109 

110def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]: 

111 """ 

112 Wrapper around __load_data that retries with key="" if anything goes wrong. 

113 """ 

114 if data is None: 

115 # try to load pyproject.toml 

116 data = find_pyproject_toml() 

117 

118 try: 

119 return __load_data(data, key, classname, lower_keys=lower_keys) 

120 except Exception as e: 

121 if key != "": 

122 return __load_data(data, "", classname, lower_keys=lower_keys) 

123 else: # pragma: no cover 

124 warnings.warn(f"Data could not be loaded: {e}", source=e) 

125 # key already was "", just return data! 

126 # (will probably not happen but fallback) 

127 return {} 

128 

129 

130F = typing.TypeVar("F") 

131 

132 

133def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T: 

134 """ 

135 Convert a value between types. 

136 """ 

137 if converter := CONVERTERS.get((from_type, to_type)): 

138 return typing.cast(T, converter(from_value)) 

139 

140 # default: just convert type: 

141 return to_type(from_value) # type: ignore 

142 

143 

144def ensure_types( 

145 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False 

146) -> dict[str, T | None]: 

147 """ 

148 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

149 

150 If an annotated key in missing from data, it will be filled with None for convenience. 

151 

152 TODO: python 3.11 exception groups to throw multiple errors at once! 

153 """ 

154 # custom object to use instead of None, since typing.Optional can be None! 

155 # cast to T to make mypy happy 

156 notfound = typing.cast(T, object()) 

157 postponed = Postponed() 

158 

159 final: dict[str, T | None] = {} 

160 for key, _type in annotations.items(): 

161 compare = data.get(key, notfound) 

162 if compare is notfound: # pragma: nocover 

163 warnings.warn( 

164 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

165 ) 

166 # skip! 

167 continue 

168 

169 if compare is postponed: 

170 # don't do anything with this item! 

171 continue 

172 

173 if not check_type(compare, _type): 

174 if convert_types: 

175 try: 

176 compare = convert_between(compare, type(compare), _type) 

177 except (TypeError, ValueError) as e: 

178 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e 

179 else: 

180 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

181 

182 final[key] = compare 

183 

184 return final 

185 

186 

187def convert_config(items: dict[str, T]) -> dict[str, T]: 

188 """ 

189 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

190 

191 1. removes any items where the value is None, since in that case the default should be used; 

192 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

193 """ 

194 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

195 

196 

197Type = typing.Type[typing.Any] 

198T_Type = typing.TypeVar("T_Type", bound=Type) 

199 

200 

201def load_recursive( 

202 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False 

203) -> dict[str, T]: 

204 """ 

205 For all annotations (recursively gathered from parents with `all_annotations`), \ 

206 try to resolve the tree of annotations. 

207 

208 Uses `load_into_recurse`, not itself directly. 

209 

210 Example: 

211 class First: 

212 key: str 

213 

214 class Second: 

215 other: First 

216 

217 # step 1 

218 cls = Second 

219 data = {"second": {"other": {"key": "anything"}}} 

220 annotations: {"other": First} 

221 

222 # step 1.5 

223 data = {"other": {"key": "anything"} 

224 annotations: {"other": First} 

225 

226 # step 2 

227 cls = First 

228 data = {"key": "anything"} 

229 annotations: {"key": str} 

230 

231 

232 TODO: python 3.11 exception groups to throw multiple errors at once! 

233 """ 

234 updated = {} 

235 

236 for _key, _type in annotations.items(): 

237 if _key in data: 

238 value: typing.Any = data[_key] # value can change so define it as any instead of T 

239 if is_parameterized(_type): 

240 origin = typing.get_origin(_type) 

241 arguments = typing.get_args(_type) 

242 if origin is list and arguments and is_custom_class(arguments[0]): 

243 subtype = arguments[0] 

244 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value] 

245 

246 elif origin is dict and arguments and is_custom_class(arguments[1]): 

247 # e.g. dict[str, Point] 

248 subkeytype, subvaluetype = arguments 

249 # subkey(type) is not a custom class, so don't try to convert it: 

250 value = { 

251 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types) 

252 for subkey, subvalue in value.items() 

253 } 

254 # elif origin is dict: 

255 # keep data the same 

256 elif origin is typing.Union and arguments: 

257 for arg in arguments: 

258 if is_custom_class(arg): 

259 value = _load_into_recurse(arg, value, convert_types=convert_types) 

260 else: 

261 # print(_type, arg, value) 

262 ... 

263 

264 # todo: other parameterized/unions/typing.Optional 

265 

266 elif is_custom_class(_type): 

267 # type must be C (custom class) at this point 

268 value = _load_into_recurse( 

269 # make mypy and pycharm happy by telling it _type is of type C... 

270 # actually just passing _type as first arg! 

271 typing.cast(Type_C[typing.Any], _type), 

272 value, 

273 convert_types=convert_types, 

274 ) 

275 

276 elif _key in cls.__dict__: 

277 # property has default, use that instead. 

278 value = cls.__dict__[_key] 

279 elif is_optional(_type): 

280 # type is optional and not found in __dict__ -> default is None 

281 value = None 

282 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

283 # could have a default factory 

284 # todo: do something with field.default? 

285 value = field.default_factory() 

286 else: 

287 raise ConfigErrorMissingKey(_key, cls, _type) 

288 

289 updated[_key] = value 

290 

291 return updated 

292 

293 

294def check_and_convert_data( 

295 cls: typing.Type[C], 

296 data: dict[str, typing.Any], 

297 _except: typing.Iterable[str], 

298 strict: bool = True, 

299 convert_types: bool = False, 

300) -> dict[str, typing.Any]: 

301 """ 

302 Based on class annotations, this prepares the data for `load_into_recurse`. 

303 

304 1. convert config-keys to python compatible config_keys 

305 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

306 3. ensures the annotated types match the actual types after loading the config file. 

307 """ 

308 annotations = all_annotations(cls, _except=_except) 

309 

310 to_load = convert_config(data) 

311 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types) 

312 if strict: 

313 to_load = ensure_types(to_load, annotations, convert_types=convert_types) 

314 

315 return to_load 

316 

317 

318T_init_list = list[typing.Any] 

319T_init_dict = dict[str, typing.Any] 

320T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

321 

322 

323@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

324def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

325 """ 

326 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

327 """ 

328 if not init: 

329 return [], {} 

330 

331 args: T_init_list = [] 

332 kwargs: T_init_dict = {} 

333 match init: 

334 case (args, kwargs): 

335 return args, kwargs 

336 case [*args]: 

337 return args, {} 

338 case {**kwargs}: 

339 return [], kwargs 

340 case _: 

341 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

342 

343 

344def _load_into_recurse( 

345 cls: typing.Type[C], 

346 data: dict[str, typing.Any], 

347 init: T_init = None, 

348 strict: bool = True, 

349 convert_types: bool = False, 

350) -> C: 

351 """ 

352 Loads an instance of `cls` filled with `data`. 

353 

354 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

355 `init` can be used to optionally pass extra __init__ arguments. \ 

356 NOTE: This will overwrite a config key with the same name! 

357 """ 

358 init_args, init_kwargs = _split_init(init) 

359 

360 if issubclass(cls, BinaryConfig): 

361 # todo: init? 

362 

363 if not isinstance(data, (bytes, dict)): # pragma: no cover 

364 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.") 

365 inst = typing.cast(C, cls._parse_into(data)) 

366 elif dc.is_dataclass(cls): 

367 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types) 

368 if init: 

369 raise ValueError("Init is not allowed for dataclasses!") 

370 

371 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

372 inst = typing.cast(C, cls(**to_load)) 

373 else: 

374 inst = cls(*init_args, **init_kwargs) 

375 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types) 

376 inst.__dict__.update(**to_load) 

377 

378 return inst 

379 

380 

381def _load_into_instance( 

382 inst: C, 

383 cls: typing.Type[C], 

384 data: dict[str, typing.Any], 

385 init: T_init = None, 

386 strict: bool = True, 

387 convert_types: bool = False, 

388) -> C: 

389 """ 

390 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

391 and thus does not support init. 

392 

393 """ 

394 if init is not None: 

395 raise ValueError("Can not init an existing instance!") 

396 

397 existing_data = inst.__dict__ 

398 

399 to_load = check_and_convert_data( 

400 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types 

401 ) 

402 

403 inst.__dict__.update(**to_load) 

404 

405 return inst 

406 

407 

408def load_into_class( 

409 cls: typing.Type[C], 

410 data: T_data, 

411 /, 

412 key: str = None, 

413 init: T_init = None, 

414 strict: bool = True, 

415 lower_keys: bool = False, 

416 convert_types: bool = False, 

417) -> C: 

418 """ 

419 Shortcut for _load_data + load_into_recurse. 

420 """ 

421 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys) 

422 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types) 

423 

424 

425def load_into_instance( 

426 inst: C, 

427 data: T_data, 

428 /, 

429 key: str = None, 

430 init: T_init = None, 

431 strict: bool = True, 

432 lower_keys: bool = False, 

433 convert_types: bool = False, 

434) -> C: 

435 """ 

436 Shortcut for _load_data + load_into_existing. 

437 """ 

438 cls = inst.__class__ 

439 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys) 

440 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types) 

441 

442 

443def load_into( 

444 cls: typing.Type[C], 

445 data: T_data = None, 

446 /, 

447 key: str = None, 

448 init: T_init = None, 

449 strict: bool = True, 

450 lower_keys: bool = False, 

451 convert_types: bool = False, 

452) -> C: 

453 """ 

454 Load your config into a class (instance). 

455 

456 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

457 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

458 

459 Args: 

460 cls: either a class or an existing instance of that class. 

461 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

462 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

463 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

464 strict: enable type checks or allow anything? 

465 lower_keys: should the config keys be lowercased? (for .env) 

466 convert_types: should the types be converted to the annotated type if not yet matching? (for .env) 

467 

468 """ 

469 if not isinstance(cls, type): 

470 # would not be supported according to mypy, but you can still load_into(instance) 

471 return load_into_instance( 

472 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

473 ) 

474 

475 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

476 # _cls = typing.cast(typing.Type[C], cls) 

477 return load_into_class( 

478 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

479 )