Coverage for src/configuraptor/core.py: 100%

150 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-14 15:57 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import types 

6import typing 

7import warnings 

8from collections import ChainMap 

9import dataclasses as dc 

10from pathlib import Path 

11 

12from typeguard import TypeCheckError 

13from typeguard import check_type as _check_type 

14 

15from . import loaders 

16from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey 

17from .helpers import camel_to_snake 

18 

19# T is a reusable typevar 

20T = typing.TypeVar("T") 

21# t_typelike is anything that can be type hinted 

22T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

23# t_data is anything that can be fed to _load_data 

24T_data = str | Path | dict[str, typing.Any] 

25# c = a config class instance, can be any (user-defined) class 

26C = typing.TypeVar("C") 

27# type c is a config class 

28Type_C = typing.Type[C] 

29 

30 

31def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

32 """ 

33 If a key contains a dot, traverse the raw dict until the right key was found. 

34 

35 Example: 

36 key = some.nested.key 

37 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

38 -> {"with": "data"} 

39 """ 

40 parts = key.split(".") 

41 while parts: 

42 raw = raw[parts.pop(0)] 

43 

44 return raw 

45 

46 

47def _guess_key(clsname: str) -> str: 

48 """ 

49 If no key is manually defined for `load_into`, \ 

50 the class' name is converted to snake_case to use as the default key. 

51 """ 

52 return camel_to_snake(clsname) 

53 

54 

55def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

56 """ 

57 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

58 

59 E.g. class Tool will be mapped to key tool. 

60 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

61 """ 

62 if isinstance(data, str): 

63 data = Path(data) 

64 if isinstance(data, Path): 

65 with data.open("rb") as f: 

66 loader = loaders.get(data.suffix) 

67 data = loader(f) 

68 

69 if not data: 

70 return {} 

71 

72 if key is None: 

73 # try to guess key by grabbing the first one or using the class name 

74 if len(data) == 1: 

75 key = list(data.keys())[0] 

76 elif classname is not None: 

77 key = _guess_key(classname) 

78 

79 if key: 

80 return _data_for_nested_key(key, data) 

81 else: 

82 # no key found, just return all data 

83 return data 

84 

85 

86def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

87 """ 

88 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

89 

90 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

91 """ 

92 try: 

93 _check_type(value, expected_type) 

94 return True 

95 except TypeCheckError: 

96 return False 

97 

98 

99def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]: 

100 """ 

101 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

102 

103 If an annotated key in missing from data, it will be filled with None for convenience. 

104 """ 

105 # custom object to use instead of None, since typing.Optional can be None! 

106 # cast to T to make mypy happy 

107 notfound = typing.cast(T, object()) 

108 

109 final: dict[str, T | None] = {} 

110 for key, _type in annotations.items(): 

111 compare = data.get(key, notfound) 

112 if compare is notfound: # pragma: nocover 

113 warnings.warn( 

114 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

115 ) 

116 # skip! 

117 continue 

118 if not check_type(compare, _type): 

119 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

120 

121 final[key] = compare 

122 return final 

123 

124 

125def convert_config(items: dict[str, T]) -> dict[str, T]: 

126 """ 

127 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

128 

129 1. removes any items where the value is None, since in that case the default should be used; 

130 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

131 """ 

132 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

133 

134 

135Type = typing.Type[typing.Any] 

136T_Type = typing.TypeVar("T_Type", bound=Type) 

137 

138 

139def is_builtin_type(_type: Type) -> bool: 

140 """ 

141 Returns whether _type is one of the builtin types. 

142 """ 

143 return _type.__module__ in ("__builtin__", "builtins") 

144 

145 

146# def is_builtin_class_instance(obj: typing.Any) -> bool: 

147# return is_builtin_type(obj.__class__) 

148 

149 

150def is_from_types_or_typing(_type: Type) -> bool: 

151 """ 

152 Returns whether _type is one of the stlib typing/types types. 

153 

154 e.g. types.UnionType or typing.Union 

155 """ 

156 return _type.__module__ in ("types", "typing") 

157 

158 

159def is_from_other_toml_supported_module(_type: Type) -> bool: 

160 """ 

161 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

162 so this returns whether _type is a type from these stdlib modules. 

163 """ 

164 return _type.__module__ in ("datetime", "math") 

165 

166 

167def is_parameterized(_type: Type) -> bool: 

168 """ 

169 Returns whether _type is a parameterized type. 

170 

171 Examples: 

172 list[str] -> True 

173 str -> False 

174 """ 

175 return typing.get_origin(_type) is not None 

176 

177 

178def is_custom_class(_type: Type) -> bool: 

179 """ 

180 Tries to guess if _type is a builtin or a custom (user-defined) class. 

181 

182 Other logic in this module depends on knowing that. 

183 """ 

184 return ( 

185 type(_type) is type 

186 and not is_builtin_type(_type) 

187 and not is_from_other_toml_supported_module(_type) 

188 and not is_from_types_or_typing(_type) 

189 ) 

190 

191 

192def is_optional(_type: Type | None) -> bool: 

193 """ 

194 Tries to guess if _type could be optional. 

195 

196 Examples: 

197 None -> True 

198 NoneType -> True 

199 typing.Union[str, None] -> True 

200 str | None -> True 

201 list[str | None] -> False 

202 list[str] -> False 

203 """ 

204 if _type and is_parameterized(_type) and typing.get_origin(_type) in (dict, list): 

205 # e.g. list[str] 

206 # will crash issubclass to test it first here 

207 return False 

208 

209 return ( 

210 _type is None 

211 or issubclass(types.NoneType, _type) 

212 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

213 or type(None) in typing.get_args(_type) # union with Nonetype 

214 ) 

215 

216 

217def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field]: 

218 """ 

219 Get Field info for a dataclass cls. 

220 """ 

221 fields = getattr(cls, "__dataclass_fields__", {}) 

222 return fields.get(key) 

223 

224 

225def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]: 

226 """ 

227 For all annotations (recursively gathered from parents with `all_annotations`), \ 

228 try to resolve the tree of annotations. 

229 

230 Uses `load_into_recurse`, not itself directly. 

231 

232 Example: 

233 class First: 

234 key: str 

235 

236 class Second: 

237 other: First 

238 

239 # step 1 

240 cls = Second 

241 data = {"second": {"other": {"key": "anything"}}} 

242 annotations: {"other": First} 

243 

244 # step 1.5 

245 data = {"other": {"key": "anything"} 

246 annotations: {"other": First} 

247 

248 # step 2 

249 cls = First 

250 data = {"key": "anything"} 

251 annotations: {"key": str} 

252 

253 """ 

254 

255 updated = {} 

256 for _key, _type in annotations.items(): 

257 

258 if _key in data: 

259 value: typing.Any = data[_key] # value can change so define it as any instead of T 

260 if is_parameterized(_type): 

261 origin = typing.get_origin(_type) 

262 arguments = typing.get_args(_type) 

263 if origin is list and arguments and is_custom_class(arguments[0]): 

264 subtype = arguments[0] 

265 value = [load_into_recurse(subtype, subvalue) for subvalue in value] 

266 

267 elif origin is dict and arguments and is_custom_class(arguments[1]): 

268 # e.g. dict[str, Point] 

269 subkeytype, subvaluetype = arguments 

270 # subkey(type) is not a custom class, so don't try to convert it: 

271 value = {subkey: load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()} 

272 # elif origin is dict: 

273 # keep data the same 

274 elif origin is typing.Union and arguments: 

275 for arg in arguments: 

276 if is_custom_class(arg): 

277 value = load_into_recurse(arg, value) 

278 else: 

279 # print(_type, arg, value) 

280 ... 

281 

282 # todo: other parameterized/unions/typing.Optional 

283 

284 elif is_custom_class(_type): 

285 # type must be C (custom class) at this point 

286 value = load_into_recurse( 

287 # make mypy and pycharm happy by telling it _type is of type C... 

288 # actually just passing _type as first arg! 

289 typing.cast(Type_C[typing.Any], _type), 

290 value, 

291 ) 

292 

293 elif _key in cls.__dict__: 

294 # property has default, use that instead. 

295 value = cls.__dict__[_key] 

296 elif is_optional(_type): 

297 # type is optional and not found in __dict__ -> default is None 

298 value = None 

299 elif ( 

300 dc.is_dataclass(cls) 

301 and (field := dataclass_field(cls, _key)) 

302 and field.default_factory is not dc.MISSING 

303 ): 

304 # could have a default factory 

305 # todo: do something with field.default? 

306 value = field.default_factory() 

307 else: 

308 # todo: exception group? 

309 raise ConfigErrorMissingKey(_key, cls, _type) 

310 

311 updated[_key] = value 

312 

313 return updated 

314 

315 

316def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

317 """ 

318 Returns a dictionary-like ChainMap that includes annotations for all \ 

319 attributes defined in cls or inherited from superclasses. 

320 """ 

321 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

322 

323 

324def all_annotations(cls: Type, _except: typing.Iterable[str]) -> dict[str, Type]: 

325 """ 

326 Wrapper around `_all_annotations` that filters away any keys in _except. 

327 

328 It also flattens the ChainMap to a regular dict. 

329 """ 

330 _all = _all_annotations(cls) 

331 return {k: v for k, v in _all.items() if k not in _except} 

332 

333 

334def _check_and_convert_data( 

335 cls: typing.Type[C], 

336 data: dict[str, typing.Any], 

337 _except: typing.Iterable[str], 

338) -> dict[str, typing.Any]: 

339 """ 

340 Based on class annotations, this prepares the data for `load_into_recurse`. 

341 

342 1. convert config-keys to python compatible config_keys 

343 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

344 3. ensures the annotated types match the actual types after loading the config file. 

345 """ 

346 annotations = all_annotations(cls, _except=_except) 

347 

348 to_load = convert_config(data) 

349 to_load = load_recursive(cls, to_load, annotations) 

350 to_load = ensure_types(to_load, annotations) 

351 return to_load 

352 

353 

354def load_into_recurse( 

355 cls: typing.Type[C], 

356 data: dict[str, typing.Any], 

357 init: dict[str, typing.Any] = None, 

358) -> C: 

359 """ 

360 Loads an instance of `cls` filled with `data`. 

361 

362 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

363 `init` can be used to optionally pass extra __init__ arguments. \ 

364 NOTE: This will overwrite a config key with the same name! 

365 """ 

366 if init is None: 

367 init = {} 

368 

369 # fixme: cls.__init__ can set other keys than the name is in kwargs!! 

370 

371 if dc.is_dataclass(cls): 

372 to_load = _check_and_convert_data(cls, data, init.keys()) 

373 to_load |= init # add extra init variables (should not happen for a dataclass but whatev) 

374 

375 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

376 inst = typing.cast(C, cls(**to_load)) 

377 else: 

378 inst = cls(**init) 

379 to_load = _check_and_convert_data(cls, data, inst.__dict__.keys()) 

380 inst.__dict__.update(**to_load) 

381 

382 return inst 

383 

384 

385def load_into_existing( 

386 inst: C, 

387 cls: typing.Type[C], 

388 data: dict[str, typing.Any], 

389 init: dict[str, typing.Any] = None, 

390) -> C: 

391 """ 

392 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

393 and thus does not support init. 

394 

395 """ 

396 if init is not None: 

397 raise ValueError("Can not init an existing instance!") 

398 

399 existing_data = inst.__dict__ 

400 

401 annotations = all_annotations(cls, _except=existing_data.keys()) 

402 to_load = convert_config(data) 

403 to_load = load_recursive(cls, to_load, annotations) 

404 to_load = ensure_types(to_load, annotations) 

405 

406 inst.__dict__.update(**to_load) 

407 

408 return inst 

409 

410 

411def load_into_class( 

412 cls: typing.Type[C], 

413 data: T_data, 

414 /, 

415 key: str = None, 

416 init: dict[str, typing.Any] = None, 

417) -> C: 

418 """ 

419 Shortcut for _load_data + load_into_recurse. 

420 """ 

421 to_load = _load_data(data, key, cls.__name__) 

422 return load_into_recurse(cls, to_load, init=init) 

423 

424 

425def load_into_instance( 

426 inst: C, 

427 data: T_data, 

428 /, 

429 key: str = None, 

430 init: dict[str, typing.Any] = None, 

431) -> C: 

432 """ 

433 Shortcut for _load_data + load_into_existing. 

434 """ 

435 cls = inst.__class__ 

436 to_load = _load_data(data, key, cls.__name__) 

437 return load_into_existing(inst, cls, to_load, init=init) 

438 

439 

440def load_into( 

441 cls: typing.Type[C] | C, 

442 data: T_data, 

443 /, 

444 key: str = None, 

445 init: dict[str, typing.Any] = None, 

446) -> C: 

447 """ 

448 Load your config into a class (instance). 

449 

450 Args: 

451 cls: either a class or an existing instance of that class. 

452 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

453 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

454 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

455 

456 """ 

457 if not isinstance(cls, type): 

458 return load_into_instance(cls, data, key=key, init=init) 

459 

460 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

461 _cls = typing.cast(typing.Type[C], cls) 

462 return load_into_class(_cls, data, key=key, init=init)