Coverage for src/configuraptor/core.py: 100%

219 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2024-01-22 21:21 +0100

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import io 

7import os 

8import typing 

9import warnings 

10from pathlib import Path 

11from typing import Any, Type 

12 

13import requests 

14 

15from . import loaders 

16from .abs import AnyType, C, T, T_data, Type_C 

17from .alias import Alias, has_alias 

18from .binary_config import BinaryConfig 

19from .errors import ( 

20 ConfigErrorCouldNotConvert, 

21 ConfigErrorInvalidType, 

22 ConfigErrorMissingKey, 

23) 

24from .helpers import ( 

25 all_annotations, 

26 camel_to_snake, 

27 check_type, 

28 dataclass_field, 

29 find_pyproject_toml, 

30 is_custom_class, 

31 is_optional, 

32 is_parameterized, 

33) 

34from .postpone import Postponed 

35from .type_converters import CONVERTERS 

36 

37 

38def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

39 """ 

40 If a key contains a dot, traverse the raw dict until the right key was found. 

41 

42 Example: 

43 key = some.nested.key 

44 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

45 -> {"with": "data"} 

46 """ 

47 parts = key.split(".") 

48 while parts: 

49 key = parts.pop(0) 

50 if key not in raw: 

51 return {} 

52 

53 raw = raw[key] 

54 

55 return raw 

56 

57 

58def _guess_key(clsname: str) -> str: 

59 """ 

60 If no key is manually defined for `load_into`, \ 

61 the class' name is converted to snake_case to use as the default key. 

62 """ 

63 return camel_to_snake(clsname) 

64 

65 

66def _from_mock_url(url: str) -> str: 

67 """ 

68 Pytest only: when starting a url with mock:// it is expected to just be json afterwards. 

69 """ 

70 return url.removeprefix("mock://") 

71 

72 

73def guess_filetype_for_url(url: str, response: requests.Response = None) -> str: 

74 """ 

75 Based on the url (which may have an extension) and the requests response \ 

76 (which may have a content-type), try to guess the right filetype (-> loader, e.g. json or yaml). 

77 

78 Falls back to JSON if none can be found. 

79 """ 

80 url = url.split("?")[0] 

81 if url_extension := os.path.splitext(url)[1].lower(): 

82 return url_extension.strip(".") 

83 

84 if response and (content_type_header := response.headers.get("content-type", "").split(";")[0].strip()): 

85 content_type = content_type_header.split("/")[-1] 

86 if content_type != "plain": 

87 return content_type 

88 

89 # If both methods fail, default to JSON 

90 return "json" 

91 

92 

93def from_url(url: str, _dummy: bool = False) -> tuple[io.BytesIO, str]: 

94 """ 

95 Load data as bytes into a file-like object and return the file type. 

96 

97 This can be used by __load_data: 

98 > loader = loaders.get(filetype) 

99 > # dev/null exists but always returns b'' 

100 > data = loader(contents, Path("/dev/null")) 

101 """ 

102 if url.startswith("mock://"): 

103 data = _from_mock_url(url) 

104 resp = None 

105 elif _dummy: 

106 resp = None 

107 data = "{}" 

108 else: 

109 resp = requests.get(url, timeout=10) 

110 data = resp.text 

111 

112 filetype = guess_filetype_for_url(url, resp) 

113 return io.BytesIO(data.encode()), filetype 

114 

115 

116def _load_data( 

117 data: T_data, 

118 key: str = None, 

119 classname: str = None, 

120 lower_keys: bool = False, 

121 allow_types: tuple[type, ...] = (dict,), 

122) -> dict[str, typing.Any]: 

123 """ 

124 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

125 

126 E.g. class Tool will be mapped to key tool. 

127 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

128 """ 

129 if isinstance(data, bytes): 

130 # instantly return, don't modify 

131 # bytes as inputs -> bytes as output 

132 # but since `T_data` is re-used, that's kind of hard to type for mypy. 

133 return data # type: ignore 

134 

135 if isinstance(data, list): 

136 if not data: 

137 raise ValueError("Empty list passed!") 

138 

139 final_data: dict[str, typing.Any] = {} 

140 for source in data: 

141 final_data |= load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types) 

142 

143 return final_data 

144 

145 if isinstance(data, str): 

146 if data.startswith(("http://", "https://", "mock://")): 

147 contents, filetype = from_url(data) 

148 

149 loader = loaders.get(filetype) 

150 # dev/null exists but always returns b'' 

151 data = loader(contents, Path("/dev/null")) 

152 else: 

153 data = Path(data) 

154 

155 if isinstance(data, Path): 

156 with data.open("rb") as f: 

157 loader = loaders.get(data.suffix or data.name) 

158 data = loader(f, data.resolve()) 

159 

160 if not data: 

161 return {} 

162 

163 if key is None: 

164 # try to guess key by grabbing the first one or using the class name 

165 if len(data) == 1: 

166 key = next(iter(data.keys())) 

167 elif classname is not None: 

168 key = _guess_key(classname) 

169 

170 if key: 

171 data = _data_for_nested_key(key, data) 

172 

173 if not data: 

174 raise ValueError("No data found!") 

175 

176 if not isinstance(data, allow_types): 

177 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!") 

178 

179 if lower_keys and isinstance(data, dict): 

180 data = {k.lower(): v for k, v in data.items()} 

181 

182 return typing.cast(dict[str, typing.Any], data) 

183 

184 

185def load_data( 

186 data: T_data, 

187 key: str = None, 

188 classname: str = None, 

189 lower_keys: bool = False, 

190 allow_types: tuple[type, ...] = (dict,), 

191) -> dict[str, typing.Any]: 

192 """ 

193 Wrapper around __load_data that retries with key="" if anything goes wrong. 

194 """ 

195 if data is None: 

196 # try to load pyproject.toml 

197 data = find_pyproject_toml() 

198 

199 try: 

200 return _load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types) 

201 except Exception as e: 

202 # sourcery skip: remove-unnecessary-else, simplify-empty-collection-comparison, swap-if-else-branches 

203 # @sourcery: `key != ""` is NOT the same as `not key` 

204 if key != "": 

205 return _load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types) 

206 else: # pragma: no cover 

207 warnings.warn(f"Data could not be loaded: {e}", source=e) 

208 # key already was "", just return data! 

209 # (will probably not happen but fallback) 

210 return {} 

211 

212 

213F = typing.TypeVar("F") 

214 

215 

216def convert_between(from_value: F, from_type: Type[F], to_type: Type[T]) -> T: 

217 """ 

218 Convert a value between types. 

219 """ 

220 if converter := CONVERTERS.get((from_type, to_type)): 

221 return typing.cast(T, converter(from_value)) 

222 

223 # default: just convert type: 

224 return to_type(from_value) # type: ignore 

225 

226 

227def check_and_convert_type(value: Any, _type: Type[T], convert_types: bool, key: str = "variable") -> T: 

228 """ 

229 Checks if the given value matches the specified type. If it does, the value is returned as is. 

230 

231 Args: 

232 value (Any): The value to be checked and potentially converted. 

233 _type (Type[T]): The expected type for the value. 

234 convert_types (bool): If True, allows type conversion if the types do not match. 

235 key (str, optional): The name or key associated with the variable (used in error messages). 

236 Defaults to "variable". 

237 

238 Returns: 

239 T: The value, potentially converted to the expected type. 

240 

241 Raises: 

242 ConfigErrorInvalidType: If the type does not match, and type conversion is not allowed. 

243 ConfigErrorCouldNotConvert: If type conversion fails. 

244 """ 

245 if check_type(value, _type): 

246 # type matches 

247 return value 

248 

249 if isinstance(value, Alias): 

250 if is_optional(_type): 

251 return typing.cast(T, None) 

252 else: 

253 # unresolved alias, error should've already been thrown for parent but lets do it again: 

254 raise ConfigErrorInvalidType(value.to, value=value, expected_type=_type) 

255 

256 if not convert_types: 

257 # type does not match and should not be converted 

258 raise ConfigErrorInvalidType(key, value=value, expected_type=_type) 

259 

260 # else: type does not match, try to convert it 

261 try: 

262 return convert_between(value, type(value), _type) 

263 except (TypeError, ValueError) as e: 

264 raise ConfigErrorCouldNotConvert(type(value), _type, value) from e 

265 

266 

267def ensure_types( 

268 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False 

269) -> dict[str, T | None]: 

270 """ 

271 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

272 

273 If an annotated key in missing from data, it will be filled with None for convenience. 

274 

275 TODO: python 3.11 exception groups to throw multiple errors at once! 

276 """ 

277 # custom object to use instead of None, since typing.Optional can be None! 

278 # cast to T to make mypy happy 

279 notfound = typing.cast(T, object()) 

280 

281 final: dict[str, T | None] = {} 

282 for key, _type in annotations.items(): 

283 compare = data.get(key, notfound) 

284 if compare is notfound: # pragma: nocover 

285 warnings.warn( 

286 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

287 ) 

288 # skip! 

289 continue 

290 

291 if isinstance(compare, Postponed): 

292 # don't do anything with this item! 

293 continue 

294 

295 if isinstance(compare, Alias): 

296 related_data = data.get(compare.to, notfound) 

297 if related_data is not notfound: 

298 if isinstance(related_data, Postponed): 

299 # also continue alias for postponed items 

300 continue 

301 

302 # original key set, update alias 

303 compare = related_data 

304 

305 compare = check_and_convert_type(compare, _type, convert_types, key) 

306 

307 final[key] = compare 

308 

309 return final 

310 

311 

312def convert_key(key: str) -> str: 

313 """ 

314 Replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

315 """ 

316 return key.replace("-", "_").replace(".", "_") 

317 

318 

319def convert_config(items: dict[str, T]) -> dict[str, T]: 

320 """ 

321 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

322 

323 1. removes any items where the value is None, since in that case the default should be used; 

324 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

325 """ 

326 return {convert_key(k): v for k, v in items.items() if v is not None} 

327 

328 

329def load_recursive( 

330 cls: AnyType, data: dict[str, T], annotations: dict[str, AnyType], convert_types: bool = False 

331) -> dict[str, T]: 

332 """ 

333 For all annotations (recursively gathered from parents with `all_annotations`), \ 

334 try to resolve the tree of annotations. 

335 

336 Uses `load_into_recurse`, not itself directly. 

337 

338 Example: 

339 class First: 

340 key: str 

341 

342 class Second: 

343 other: First 

344 

345 # step 1 

346 cls = Second 

347 data = {"second": {"other": {"key": "anything"}}} 

348 annotations: {"other": First} 

349 

350 # step 1.5 

351 data = {"other": {"key": "anything"} 

352 annotations: {"other": First} 

353 

354 # step 2 

355 cls = First 

356 data = {"key": "anything"} 

357 annotations: {"key": str} 

358 

359 

360 TODO: python 3.11 exception groups to throw multiple errors at once! 

361 """ 

362 updated = {} 

363 

364 for _key, _type in annotations.items(): 

365 if _key in data: 

366 value: typing.Any = data[_key] # value can change so define it as any instead of T 

367 if is_parameterized(_type): 

368 origin = typing.get_origin(_type) 

369 arguments = typing.get_args(_type) 

370 if origin is list and arguments and is_custom_class(arguments[0]): 

371 subtype = arguments[0] 

372 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value] 

373 

374 elif origin is dict and arguments and is_custom_class(arguments[1]): 

375 # e.g. dict[str, Point] 

376 subkeytype, subvaluetype = arguments 

377 # subkey(type) is not a custom class, so don't try to convert it: 

378 value = { 

379 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types) 

380 for subkey, subvalue in value.items() 

381 } 

382 # elif origin is dict: 

383 # keep data the same 

384 elif origin is typing.Union and arguments: 

385 for arg in arguments: 

386 if is_custom_class(arg): 

387 value = _load_into_recurse(arg, value, convert_types=convert_types) 

388 

389 # todo: other parameterized/unions/typing.Optional 

390 

391 elif is_custom_class(_type): 

392 # type must be C (custom class) at this point 

393 value = _load_into_recurse( 

394 # make mypy and pycharm happy by telling it _type is of type C... 

395 # actually just passing _type as first arg! 

396 typing.cast(Type_C[typing.Any], _type), 

397 value, 

398 convert_types=convert_types, 

399 ) 

400 elif value := has_alias(cls, _key, data): 

401 # value updated by alias 

402 ... 

403 elif _key in cls.__dict__: 

404 # property has default, use that instead. 

405 value = cls.__dict__[_key] 

406 elif is_optional(_type): 

407 # type is optional and not found in __dict__ -> default is None 

408 value = None 

409 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

410 # could have a default factory 

411 # todo: do something with field.default? 

412 value = field.default_factory() 

413 else: 

414 raise ConfigErrorMissingKey(_key, cls, _type) 

415 

416 updated[_key] = value 

417 

418 return updated 

419 

420 

421def check_and_convert_data( 

422 cls: typing.Type[C], 

423 data: dict[str, typing.Any], 

424 _except: typing.Iterable[str], 

425 strict: bool = True, 

426 convert_types: bool = False, 

427) -> dict[str, typing.Any]: 

428 """ 

429 Based on class annotations, this prepares the data for `load_into_recurse`. 

430 

431 1. convert config-keys to python compatible config_keys 

432 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

433 3. ensures the annotated types match the actual types after loading the config file. 

434 """ 

435 annotations = all_annotations(cls, _except=_except) 

436 

437 to_load = convert_config(data) 

438 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types) 

439 

440 if strict: 

441 to_load = ensure_types(to_load, annotations, convert_types=convert_types) 

442 

443 return to_load 

444 

445 

446T_init_list = list[typing.Any] 

447T_init_dict = dict[str, typing.Any] 

448T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None 

449 

450 

451@typing.no_type_check # (mypy doesn't understand 'match' fully yet) 

452def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]: 

453 """ 

454 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple. 

455 """ 

456 if not init: 

457 return [], {} 

458 

459 args: T_init_list = [] 

460 kwargs: T_init_dict = {} 

461 match init: 

462 case (args, kwargs): 

463 return args, kwargs 

464 case [*args]: 

465 return args, {} 

466 case {**kwargs}: 

467 return [], kwargs 

468 case _: 

469 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.") 

470 

471 

472def _load_into_recurse( 

473 cls: typing.Type[C], 

474 data: dict[str, typing.Any] | bytes, 

475 init: T_init = None, 

476 strict: bool = True, 

477 convert_types: bool = False, 

478) -> C: 

479 """ 

480 Loads an instance of `cls` filled with `data`. 

481 

482 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

483 `init` can be used to optionally pass extra __init__ arguments. \ 

484 NOTE: This will overwrite a config key with the same name! 

485 """ 

486 init_args, init_kwargs = _split_init(init) 

487 

488 if isinstance(data, bytes) or issubclass(cls, BinaryConfig): 

489 if not isinstance(data, (bytes, dict)): # pragma: no cover 

490 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.") 

491 elif not issubclass(cls, BinaryConfig): # pragma: no cover 

492 raise NotImplementedError("Only BinaryConfig can be used with `bytes` (or a dict of bytes) as input.") 

493 

494 inst = typing.cast(C, cls._parse_into(data)) 

495 elif dc.is_dataclass(cls): 

496 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types) 

497 if init: 

498 raise ValueError("Init is not allowed for dataclasses!") 

499 

500 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

501 inst = typing.cast(C, cls(**to_load)) 

502 else: 

503 inst = cls(*init_args, **init_kwargs) 

504 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types) 

505 inst.__dict__.update(**to_load) 

506 

507 return inst 

508 

509 

510def _load_into_instance( 

511 inst: C, 

512 cls: typing.Type[C], 

513 data: dict[str, typing.Any], 

514 init: T_init = None, 

515 strict: bool = True, 

516 convert_types: bool = False, 

517) -> C: 

518 """ 

519 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

520 and thus does not support init. 

521 

522 """ 

523 if init is not None: 

524 raise ValueError("Can not init an existing instance!") 

525 

526 existing_data = inst.__dict__ 

527 

528 to_load = check_and_convert_data( 

529 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types 

530 ) 

531 

532 inst.__dict__.update(**to_load) 

533 

534 return inst 

535 

536 

537def load_into_class( 

538 cls: typing.Type[C], 

539 data: T_data, 

540 /, 

541 key: str = None, 

542 init: T_init = None, 

543 strict: bool = True, 

544 lower_keys: bool = False, 

545 convert_types: bool = False, 

546) -> C: 

547 """ 

548 Shortcut for _load_data + load_into_recurse. 

549 """ 

550 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,) 

551 to_load = load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types) 

552 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types) 

553 

554 

555def load_into_instance( 

556 inst: C, 

557 data: T_data, 

558 /, 

559 key: str = None, 

560 init: T_init = None, 

561 strict: bool = True, 

562 lower_keys: bool = False, 

563 convert_types: bool = False, 

564) -> C: 

565 """ 

566 Shortcut for _load_data + load_into_existing. 

567 """ 

568 cls = inst.__class__ 

569 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,) 

570 to_load = load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types) 

571 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types) 

572 

573 

574def load_into( 

575 cls: typing.Type[C], 

576 data: T_data = None, 

577 /, 

578 key: str = None, 

579 init: T_init = None, 

580 strict: bool = True, 

581 lower_keys: bool = False, 

582 convert_types: bool = False, 

583) -> C: 

584 """ 

585 Load your config into a class (instance). 

586 

587 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

588 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

589 

590 Args: 

591 cls: either a class or an existing instance of that class. 

592 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

593 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

594 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

595 strict: enable type checks or allow anything? 

596 lower_keys: should the config keys be lowercased? (for .env) 

597 convert_types: should the types be converted to the annotated type if not yet matching? (for .env) 

598 

599 """ 

600 if not isinstance(cls, type): 

601 # would not be supported according to mypy, but you can still load_into(instance) 

602 return load_into_instance( 

603 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

604 ) 

605 

606 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

607 # _cls = typing.cast(typing.Type[C], cls) 

608 return load_into_class( 

609 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types 

610 )