Coverage for src/configuraptor/core.py: 100%

158 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-15 16:44 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import math 

7import types 

8import typing 

9import warnings 

10from collections import ChainMap 

11from pathlib import Path 

12 

13from typeguard import TypeCheckError 

14from typeguard import check_type as _check_type 

15 

16from . import loaders 

17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey 

18from .helpers import camel_to_snake 

19from .postpone import Postponed 

20 

21# T is a reusable typevar 

22T = typing.TypeVar("T") 

23# t_typelike is anything that can be type hinted 

24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

25# t_data is anything that can be fed to _load_data 

26T_data = str | Path | dict[str, typing.Any] 

27# c = a config class instance, can be any (user-defined) class 

28C = typing.TypeVar("C") 

29# type c is a config class 

30Type_C = typing.Type[C] 

31 

32 

33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

34 """ 

35 If a key contains a dot, traverse the raw dict until the right key was found. 

36 

37 Example: 

38 key = some.nested.key 

39 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

40 -> {"with": "data"} 

41 """ 

42 parts = key.split(".") 

43 while parts: 

44 raw = raw[parts.pop(0)] 

45 

46 return raw 

47 

48 

49def _guess_key(clsname: str) -> str: 

50 """ 

51 If no key is manually defined for `load_into`, \ 

52 the class' name is converted to snake_case to use as the default key. 

53 """ 

54 return camel_to_snake(clsname) 

55 

56 

57def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

58 """ 

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

60 

61 E.g. class Tool will be mapped to key tool. 

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

63 """ 

64 if isinstance(data, str): 

65 data = Path(data) 

66 if isinstance(data, Path): 

67 with data.open("rb") as f: 

68 loader = loaders.get(data.suffix) 

69 data = loader(f) 

70 

71 if not data: 

72 return {} 

73 

74 if key is None: 

75 # try to guess key by grabbing the first one or using the class name 

76 if len(data) == 1: 

77 key = list(data.keys())[0] 

78 elif classname is not None: 

79 key = _guess_key(classname) 

80 

81 if key: 

82 return _data_for_nested_key(key, data) 

83 else: 

84 # no key found, just return all data 

85 return data 

86 

87 

88def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

89 """ 

90 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

91 

92 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

93 """ 

94 try: 

95 _check_type(value, expected_type) 

96 return True 

97 except TypeCheckError: 

98 return False 

99 

100 

101def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]: 

102 """ 

103 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

104 

105 If an annotated key in missing from data, it will be filled with None for convenience. 

106 """ 

107 # custom object to use instead of None, since typing.Optional can be None! 

108 # cast to T to make mypy happy 

109 notfound = typing.cast(T, object()) 

110 postponed = Postponed() 

111 

112 final: dict[str, T | None] = {} 

113 for key, _type in annotations.items(): 

114 compare = data.get(key, notfound) 

115 if compare is notfound: # pragma: nocover 

116 warnings.warn( 

117 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

118 ) 

119 # skip! 

120 continue 

121 

122 if compare is postponed: 

123 # don't do anything with this item! 

124 continue 

125 

126 if not check_type(compare, _type): 

127 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

128 

129 final[key] = compare 

130 

131 return final 

132 

133 

134def convert_config(items: dict[str, T]) -> dict[str, T]: 

135 """ 

136 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

137 

138 1. removes any items where the value is None, since in that case the default should be used; 

139 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

140 """ 

141 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

142 

143 

144Type = typing.Type[typing.Any] 

145T_Type = typing.TypeVar("T_Type", bound=Type) 

146 

147 

148def is_builtin_type(_type: Type) -> bool: 

149 """ 

150 Returns whether _type is one of the builtin types. 

151 """ 

152 return _type.__module__ in ("__builtin__", "builtins") 

153 

154 

155# def is_builtin_class_instance(obj: typing.Any) -> bool: 

156# return is_builtin_type(obj.__class__) 

157 

158 

159def is_from_types_or_typing(_type: Type) -> bool: 

160 """ 

161 Returns whether _type is one of the stlib typing/types types. 

162 

163 e.g. types.UnionType or typing.Union 

164 """ 

165 return _type.__module__ in ("types", "typing") 

166 

167 

168def is_from_other_toml_supported_module(_type: Type) -> bool: 

169 """ 

170 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

171 so this returns whether _type is a type from these stdlib modules. 

172 """ 

173 return _type.__module__ in ("datetime", "math") 

174 

175 

176def is_parameterized(_type: Type) -> bool: 

177 """ 

178 Returns whether _type is a parameterized type. 

179 

180 Examples: 

181 list[str] -> True 

182 str -> False 

183 """ 

184 return typing.get_origin(_type) is not None 

185 

186 

187def is_custom_class(_type: Type) -> bool: 

188 """ 

189 Tries to guess if _type is a builtin or a custom (user-defined) class. 

190 

191 Other logic in this module depends on knowing that. 

192 """ 

193 return ( 

194 type(_type) is type 

195 and not is_builtin_type(_type) 

196 and not is_from_other_toml_supported_module(_type) 

197 and not is_from_types_or_typing(_type) 

198 ) 

199 

200 

201def instance_of_custom_class(var: typing.Any) -> bool: 

202 """ 

203 Calls `is_custom_class` on an instance of a (possibly custom) class. 

204 """ 

205 return is_custom_class(var.__class__) 

206 

207 

208def is_optional(_type: Type | None) -> bool: 

209 """ 

210 Tries to guess if _type could be optional. 

211 

212 Examples: 

213 None -> True 

214 NoneType -> True 

215 typing.Union[str, None] -> True 

216 str | None -> True 

217 list[str | None] -> False 

218 list[str] -> False 

219 """ 

220 if _type and is_parameterized(_type) and typing.get_origin(_type) in (dict, list): 

221 # e.g. list[str] 

222 # will crash issubclass to test it first here 

223 return False 

224 elif _type is math.nan: 

225 return False 

226 

227 return ( 

228 _type is None 

229 or types.NoneType in typing.get_args(_type) # union with Nonetype 

230 or issubclass(types.NoneType, _type) 

231 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

232 ) 

233 

234 

235def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

236 """ 

237 Get Field info for a dataclass cls. 

238 """ 

239 fields = getattr(cls, "__dataclass_fields__", {}) 

240 return fields.get(key) 

241 

242 

243def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]: 

244 """ 

245 For all annotations (recursively gathered from parents with `all_annotations`), \ 

246 try to resolve the tree of annotations. 

247 

248 Uses `load_into_recurse`, not itself directly. 

249 

250 Example: 

251 class First: 

252 key: str 

253 

254 class Second: 

255 other: First 

256 

257 # step 1 

258 cls = Second 

259 data = {"second": {"other": {"key": "anything"}}} 

260 annotations: {"other": First} 

261 

262 # step 1.5 

263 data = {"other": {"key": "anything"} 

264 annotations: {"other": First} 

265 

266 # step 2 

267 cls = First 

268 data = {"key": "anything"} 

269 annotations: {"key": str} 

270 

271 """ 

272 updated = {} 

273 

274 for _key, _type in annotations.items(): 

275 if _key in data: 

276 value: typing.Any = data[_key] # value can change so define it as any instead of T 

277 if is_parameterized(_type): 

278 origin = typing.get_origin(_type) 

279 arguments = typing.get_args(_type) 

280 if origin is list and arguments and is_custom_class(arguments[0]): 

281 subtype = arguments[0] 

282 value = [_load_into_recurse(subtype, subvalue) for subvalue in value] 

283 

284 elif origin is dict and arguments and is_custom_class(arguments[1]): 

285 # e.g. dict[str, Point] 

286 subkeytype, subvaluetype = arguments 

287 # subkey(type) is not a custom class, so don't try to convert it: 

288 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()} 

289 # elif origin is dict: 

290 # keep data the same 

291 elif origin is typing.Union and arguments: 

292 for arg in arguments: 

293 if is_custom_class(arg): 

294 value = _load_into_recurse(arg, value) 

295 else: 

296 # print(_type, arg, value) 

297 ... 

298 

299 # todo: other parameterized/unions/typing.Optional 

300 

301 elif is_custom_class(_type): 

302 # type must be C (custom class) at this point 

303 value = _load_into_recurse( 

304 # make mypy and pycharm happy by telling it _type is of type C... 

305 # actually just passing _type as first arg! 

306 typing.cast(Type_C[typing.Any], _type), 

307 value, 

308 ) 

309 

310 elif _key in cls.__dict__: 

311 # property has default, use that instead. 

312 value = cls.__dict__[_key] 

313 elif is_optional(_type): 

314 # type is optional and not found in __dict__ -> default is None 

315 value = None 

316 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

317 # could have a default factory 

318 # todo: do something with field.default? 

319 value = field.default_factory() 

320 else: 

321 raise ConfigErrorMissingKey(_key, cls, _type) 

322 

323 updated[_key] = value 

324 

325 return updated 

326 

327 

328def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

329 """ 

330 Returns a dictionary-like ChainMap that includes annotations for all \ 

331 attributes defined in cls or inherited from superclasses. 

332 """ 

333 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

334 

335 

336def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]: 

337 """ 

338 Wrapper around `_all_annotations` that filters away any keys in _except. 

339 

340 It also flattens the ChainMap to a regular dict. 

341 """ 

342 if _except is None: 

343 _except = set() 

344 

345 _all = _all_annotations(cls) 

346 return {k: v for k, v in _all.items() if k not in _except} 

347 

348 

349def check_and_convert_data( 

350 cls: typing.Type[C], 

351 data: dict[str, typing.Any], 

352 _except: typing.Iterable[str], 

353 strict: bool = True, 

354) -> dict[str, typing.Any]: 

355 """ 

356 Based on class annotations, this prepares the data for `load_into_recurse`. 

357 

358 1. convert config-keys to python compatible config_keys 

359 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

360 3. ensures the annotated types match the actual types after loading the config file. 

361 """ 

362 annotations = all_annotations(cls, _except=_except) 

363 

364 to_load = convert_config(data) 

365 to_load = load_recursive(cls, to_load, annotations) 

366 if strict: 

367 to_load = ensure_types(to_load, annotations) 

368 

369 return to_load 

370 

371 

372def _load_into_recurse( 

373 cls: typing.Type[C], 

374 data: dict[str, typing.Any], 

375 init: dict[str, typing.Any] = None, 

376 strict: bool = True, 

377) -> C: 

378 """ 

379 Loads an instance of `cls` filled with `data`. 

380 

381 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

382 `init` can be used to optionally pass extra __init__ arguments. \ 

383 NOTE: This will overwrite a config key with the same name! 

384 """ 

385 if init is None: 

386 init = {} 

387 

388 # fixme: cls.__init__ can set other keys than the name is in kwargs!! 

389 

390 if dc.is_dataclass(cls): 

391 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict) 

392 to_load |= init # add extra init variables (should not happen for a dataclass but whatev) 

393 

394 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

395 inst = typing.cast(C, cls(**to_load)) 

396 else: 

397 inst = cls(**init) 

398 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict) 

399 inst.__dict__.update(**to_load) 

400 

401 return inst 

402 

403 

404def _load_into_instance( 

405 inst: C, 

406 cls: typing.Type[C], 

407 data: dict[str, typing.Any], 

408 init: dict[str, typing.Any] = None, 

409 strict: bool = True, 

410) -> C: 

411 """ 

412 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

413 and thus does not support init. 

414 

415 """ 

416 if init is not None: 

417 raise ValueError("Can not init an existing instance!") 

418 

419 existing_data = inst.__dict__ 

420 

421 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict) 

422 

423 inst.__dict__.update(**to_load) 

424 

425 return inst 

426 

427 

428def load_into_class( 

429 cls: typing.Type[C], 

430 data: T_data, 

431 /, 

432 key: str = None, 

433 init: dict[str, typing.Any] = None, 

434 strict: bool = True, 

435) -> C: 

436 """ 

437 Shortcut for _load_data + load_into_recurse. 

438 """ 

439 to_load = _load_data(data, key, cls.__name__) 

440 return _load_into_recurse(cls, to_load, init=init, strict=strict) 

441 

442 

443def load_into_instance( 

444 inst: C, 

445 data: T_data, 

446 /, 

447 key: str = None, 

448 init: dict[str, typing.Any] = None, 

449 strict: bool = True, 

450) -> C: 

451 """ 

452 Shortcut for _load_data + load_into_existing. 

453 """ 

454 cls = inst.__class__ 

455 to_load = _load_data(data, key, cls.__name__) 

456 return _load_into_instance(inst, cls, to_load, init=init, strict=strict) 

457 

458 

459def load_into( 

460 cls: typing.Type[C], 

461 data: T_data, 

462 /, 

463 key: str = None, 

464 init: dict[str, typing.Any] = None, 

465 strict: bool = True, 

466) -> C: 

467 """ 

468 Load your config into a class (instance). 

469 

470 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

471 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

472 

473 Args: 

474 cls: either a class or an existing instance of that class. 

475 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

476 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

477 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

478 strict: enable type checks or allow anything? 

479 

480 """ 

481 if not isinstance(cls, type): 

482 # would not be supported according to mypy, but you can still load_into(instance) 

483 return load_into_instance(cls, data, key=key, init=init, strict=strict) 

484 

485 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

486 # _cls = typing.cast(typing.Type[C], cls) 

487 return load_into_class(cls, data, key=key, init=init, strict=strict)