Coverage for src/typedal/config.py: 100%

132 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-08-05 19:10 +0200

1""" 

2TypeDAL can be configured by a combination of pyproject.toml (static), env (dynamic) and code (programmic). 

3""" 

4 

5import os 

6import re 

7import typing 

8import warnings 

9from collections import defaultdict 

10from pathlib import Path 

11from typing import Any, Optional 

12 

13import tomli 

14from configuraptor import TypedConfig, alias 

15from configuraptor.helpers import find_pyproject_toml 

16from dotenv import dotenv_values, find_dotenv 

17 

18from .types import AnyDict 

19 

20if typing.TYPE_CHECKING: 

21 from edwh_migrate import Config as MigrateConfig 

22 from pydal2sql.typer_support import Config as P2SConfig 

23 

24 

25class TypeDALConfig(TypedConfig): 

26 """ 

27 Unified config for TypeDAL runtime behavior and migration utilities. 

28 """ 

29 

30 # typedal: 

31 database: str 

32 dialect: str 

33 folder: str = "databases" 

34 caching: bool = True 

35 pool_size: int = 0 

36 pyproject: str 

37 connection: str = "default" 

38 

39 # pydal2sql: 

40 input: str = "" 

41 output: str = "" 

42 noop: bool = False 

43 magic: bool = True 

44 tables: Optional[list[str]] = None 

45 function: str = "define_tables" 

46 

47 # edwh-migrate: 

48 # migrate uri = database 

49 database_to_restore: Optional[str] 

50 migrate_cat_command: Optional[str] 

51 schema_version: Optional[str] 

52 redis_host: Optional[str] 

53 migrate_table: str = "typedal_implemented_features" 

54 flag_location: str 

55 create_flag_location: bool = True 

56 schema: str = "public" 

57 

58 # typedal (depends on properties above) 

59 migrate: bool = True 

60 fake_migrate: bool = False 

61 

62 # aliases: 

63 db_uri: str = alias("database") 

64 db_type: str = alias("dialect") 

65 db_folder: str = alias("folder") 

66 

67 # repr set by @beautify (by inheriting from TypedConfig) 

68 

69 def to_pydal2sql(self) -> "P2SConfig": 

70 """ 

71 Convert the config to the format required by pydal2sql. 

72 """ 

73 from pydal2sql.typer_support import Config, get_pydal2sql_config 

74 

75 if self.pyproject: # pragma: no cover 

76 project = Path(self.pyproject).read_text() 

77 

78 if "[tool.typedal]" not in project and "[tool.pydal2sql]" in project: 

79 # no typedal config, but existing p2s config: 

80 return get_pydal2sql_config(self.pyproject) 

81 

82 return Config.load( 

83 { 

84 "db_type": self.dialect, 

85 "format": "edwh-migrate", 

86 "tables": self.tables, 

87 "magic": self.magic, 

88 "function": self.function, 

89 "input": self.input, 

90 "output": self.output, 

91 "pyproject": self.pyproject, 

92 } 

93 ) 

94 

95 def to_migrate(self) -> "MigrateConfig": 

96 """ 

97 Convert the config to the format required by edwh-migrate. 

98 """ 

99 from edwh_migrate import Config, get_config 

100 

101 if self.pyproject: # pragma: no cover 

102 project = Path(self.pyproject).read_text() 

103 

104 if "[tool.typedal]" not in project and "[tool.migrate]" in project: 

105 # no typedal config, but existing p2s config: 

106 return get_config() 

107 

108 return Config.load( 

109 { 

110 "migrate_uri": self.database, 

111 "schema_version": self.schema_version, 

112 "redis_host": self.redis_host, 

113 "migrate_cat_command": self.migrate_cat_command, 

114 "database_to_restore": self.database_to_restore, 

115 "migrate_table": self.migrate_table, 

116 "flag_location": self.flag_location, 

117 "create_flag_location": self.create_flag_location, 

118 "schema": self.schema, 

119 "db_folder": self.folder, 

120 "migrations_file": self.output, 

121 } 

122 ) 

123 

124 

125def _load_toml(path: str | bool | Path | None = True) -> tuple[str, AnyDict]: 

126 """ 

127 Path can be a file, a directory, a bool or None. 

128 

129 If it is True or None, the default logic is used. 

130 If it is False, no data is loaded. 

131 if it is a directory, the pyproject.toml will be searched there. 

132 If it is a path, that file will be used. 

133 """ 

134 if path is False: 

135 toml_path = None 

136 elif path in (True, None): 

137 toml_path = find_pyproject_toml() 

138 elif (_p := Path(str(path))) and _p.is_file(): 

139 toml_path = _p 

140 else: 

141 toml_path = find_pyproject_toml(str(path)) 

142 

143 if not toml_path: 

144 # nothing to load 

145 return "", {} 

146 

147 try: 

148 with open(toml_path, "rb") as f: 

149 data = tomli.load(f) 

150 

151 return str(toml_path) or "", typing.cast(AnyDict, data["tool"]["typedal"]) 

152 except Exception as e: 

153 warnings.warn(f"Could not load typedal config toml: {e}", source=e) 

154 return str(toml_path) or "", {} 

155 

156 

157def _load_dotenv(path: str | bool | None = True) -> tuple[str, AnyDict]: 

158 fallback_data = {k.lower().removeprefix("typedal_"): v for k, v in os.environ.items()} 

159 if path is False: 

160 dotenv_path = None 

161 fallback_data = {} 

162 elif path in (True, None): 

163 dotenv_path = find_dotenv(usecwd=True) 

164 elif Path(str(path)).is_file(): 

165 dotenv_path = str(path) 

166 else: 

167 dotenv_path = str(Path(str(path)) / ".env") 

168 

169 if not dotenv_path: 

170 return "", fallback_data 

171 

172 # 1. find everything with TYPEDAL_ prefix 

173 # 2. remove that prefix 

174 # 3. format values if possible 

175 data = dotenv_values(dotenv_path) 

176 data |= os.environ # higher prio than .env 

177 

178 typedal_data = {k.lower().removeprefix("typedal_"): v for k, v in data.items()} 

179 

180 return dotenv_path, typedal_data 

181 

182 

183DB_ALIASES = { 

184 "postgresql": "postgres", 

185 "psql": "postgres", 

186 "sqlite3": "sqlite", 

187} 

188 

189 

190def get_db_for_alias(db_name: str) -> str: 

191 """ 

192 Convert a db dialect alias to the standard name. 

193 """ 

194 return DB_ALIASES.get(db_name, db_name) 

195 

196 

197DEFAULTS: dict[str, Any | typing.Callable[[AnyDict], Any]] = { 

198 "database": lambda data: data.get("db_uri") or "sqlite:memory", 

199 "dialect": lambda data: ( 

200 get_db_for_alias(data["database"].split(":")[0]) if ":" in data["database"] else data.get("db_type") 

201 ), 

202 "migrate": lambda data: not (data.get("input") or data.get("output")), 

203 "folder": lambda data: data.get("db_folder"), 

204 "flag_location": lambda data: ( 

205 f"{db_folder}/flags" if (db_folder := (data.get("folder") or data.get("db_folder"))) else "/flags" 

206 ), 

207 "pool_size": lambda data: 1 if data.get("dialect", "sqlite") == "sqlite" else 3, 

208} 

209 

210 

211def _fill_defaults(data: AnyDict, prop: str, fallback: Any = None) -> None: 

212 default = DEFAULTS.get(prop, fallback) 

213 if callable(default): 

214 default = default(data) 

215 data[prop] = default 

216 

217 

218def fill_defaults(data: AnyDict, prop: str) -> None: 

219 """ 

220 Fill missing property defaults with (calculated) sane defaults. 

221 """ 

222 if data.get(prop, None) is None: 

223 _fill_defaults(data, prop) 

224 

225 

226TRANSFORMS: dict[str, typing.Callable[[AnyDict], Any]] = { 

227 "database": lambda data: ( 

228 data["database"] 

229 if (":" in data["database"] or not data.get("dialect")) 

230 else (data["dialect"] + "://" + data["database"]) 

231 ) 

232} 

233 

234 

235def transform(data: AnyDict, prop: str) -> bool: 

236 """ 

237 After the user has chosen a value, possibly transform it. 

238 """ 

239 if fn := TRANSFORMS.get(prop): 

240 data[prop] = fn(data) 

241 return True 

242 return False 

243 

244 

245def expand_posix_vars(posix_expr: str, context: dict[str, str]) -> str: 

246 """ 

247 Replace case-insensitive POSIX and Docker Compose-like environment variables in a string with their values. 

248 

249 Args: 

250 posix_expr (str): The input string containing case-insensitive POSIX or Docker Compose-like variables. 

251 context (dict): A dictionary containing variable names and their respective values. 

252 

253 Returns: 

254 str: The string with replaced variable values. 

255 

256 See Also: 

257 https://stackoverflow.com/questions/386934/how-to-evaluate-environment-variables-into-a-string-in-python 

258 and ChatGPT 

259 """ 

260 env = defaultdict(lambda: "") 

261 for key, value in context.items(): 

262 env[key.lower()] = value 

263 

264 # Regular expression to match "${VAR:default}" pattern 

265 pattern = r"\$\{([^}]+)\}" 

266 

267 def replace_var(match: re.Match[Any]) -> str: 

268 var_with_default = match.group(1) 

269 var_name, default_value = var_with_default.split(":") if ":" in var_with_default else (var_with_default, "") 

270 return env.get(var_name.lower(), default_value) 

271 

272 return re.sub(pattern, replace_var, posix_expr) 

273 

274 

275def expand_env_vars_into_toml_values(toml: AnyDict, env: AnyDict) -> None: 

276 """ 

277 Recursively expands POSIX/Docker Compose-like environment variables in a TOML dictionary. 

278 

279 This function traverses a TOML dictionary and expands POSIX/Docker Compose-like 

280 environment variables (${VAR:default}) using values provided in the 'env' dictionary. 

281 It performs in-place modification of the 'toml' dictionary. 

282 

283 Args: 

284 toml (dict): A TOML dictionary with string values possibly containing environment variables. 

285 env (dict): A dictionary containing environment variable names and their respective values. 

286 

287 Returns: 

288 None: The function modifies the 'toml' dictionary in place. 

289 

290 Notes: 

291 The function recursively traverses the 'toml' dictionary. If a value is a string or a list of strings, 

292 it attempts to substitute any environment variables found within those strings using the 'env' dictionary. 

293 

294 Example: 

295 toml_data = { 

296 'key1': 'This has ${ENV_VAR:default}', 

297 'key2': ['String with ${ANOTHER_VAR}', 'Another ${YET_ANOTHER_VAR}'] 

298 } 

299 environment = { 

300 'ENV_VAR': 'replaced_value', 

301 'ANOTHER_VAR': 'value_1', 

302 'YET_ANOTHER_VAR': 'value_2' 

303 } 

304 

305 expand_env_vars_into_toml_values(toml_data, environment) 

306 # 'toml_data' will be modified in place: 

307 # { 

308 # 'key1': 'This has replaced_value', 

309 # 'key2': ['String with value_1', 'Another value_2'] 

310 # } 

311 """ 

312 if not toml or not env: 

313 return 

314 

315 for key, var in toml.items(): 

316 if isinstance(var, dict): 

317 expand_env_vars_into_toml_values(var, env) 

318 elif isinstance(var, list): 

319 toml[key] = [expand_posix_vars(_, env) for _ in var if isinstance(_, str)] 

320 elif isinstance(var, str): 

321 toml[key] = expand_posix_vars(var, env) 

322 else: 

323 # nothing to substitute 

324 continue 

325 

326 

327def load_config( 

328 connection_name: Optional[str] = None, 

329 _use_pyproject: bool | str | None = True, 

330 _use_env: bool | str | None = True, 

331 **fallback: Any, 

332) -> TypeDALConfig: 

333 """ 

334 Combines multiple sources of config into one config instance. 

335 """ 

336 # load toml data 

337 # load .env data 

338 # combine and fill with fallback values 

339 # load typedal config or fail 

340 toml_path, toml = _load_toml(_use_pyproject) 

341 dotenv_path, dotenv = _load_dotenv(_use_env) 

342 

343 expand_env_vars_into_toml_values(toml, dotenv) 

344 

345 connection_name = connection_name or dotenv.get("connection", "") or toml.get("default", "") 

346 connection: AnyDict = (toml.get(connection_name) if connection_name else toml) or {} 

347 

348 combined = connection | dotenv | fallback 

349 combined = {k.replace("-", "_"): v for k, v in combined.items()} 

350 

351 combined["pyproject"] = toml_path 

352 combined["connection"] = connection_name 

353 

354 for prop in TypeDALConfig.__annotations__: 

355 fill_defaults(combined, prop) 

356 

357 for prop in TypeDALConfig.__annotations__: 

358 transform(combined, prop) 

359 

360 return TypeDALConfig.load(combined, convert_types=True)