Coverage for src/typedal/config.py: 100%
133 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-14 15:15 +0100
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-14 15:15 +0100
1"""
2TypeDAL can be configured by a combination of pyproject.toml (static), env (dynamic) and code (programmic).
3"""
4import os
5import re
6import typing
7import warnings
8from collections import defaultdict
9from pathlib import Path
10from typing import Any, Optional
12import black.files
13import tomli
14from configuraptor import TypedConfig, alias
15from dotenv import dotenv_values, find_dotenv
17if typing.TYPE_CHECKING: # pragma: no cover
18 from edwh_migrate import Config as MigrateConfig
19 from pydal2sql.typer_support import Config as P2SConfig
22class TypeDALConfig(TypedConfig):
23 """
24 Unified config for TypeDAL runtime behavior and migration utilities.
25 """
27 # typedal:
28 database: str
29 dialect: str
30 folder: str = "databases"
31 caching: bool = True
32 pool_size: int = 0
33 pyproject: str
34 connection: str = "default"
36 # pydal2sql:
37 input: str = "" # noqa: A003
38 output: str = ""
39 noop: bool = False
40 magic: bool = True
41 tables: Optional[list[str]] = None
42 function: str = "define_tables"
44 # edwh-migrate:
45 # migrate uri = database
46 database_to_restore: Optional[str]
47 migrate_cat_command: Optional[str]
48 schema_version: Optional[str]
49 redis_host: Optional[str]
50 migrate_table: str = "typedal_implemented_features"
51 flag_location: str
52 create_flag_location: bool = True
53 schema: str = "public"
55 # typedal (depends on properties above)
56 migrate: bool = True
57 fake_migrate: bool = False
59 # aliases:
60 db_uri: str = alias("database")
61 db_type: str = alias("dialect")
62 db_folder: str = alias("folder")
64 def __repr__(self) -> str:
65 """
66 Dump the config to a (fancy) string.
67 """
68 return f"<TypeDAL {self.__dict__}>"
70 def to_pydal2sql(self) -> "P2SConfig":
71 """
72 Convert the config to the format required by pydal2sql.
73 """
74 from pydal2sql.typer_support import Config, get_pydal2sql_config
76 if self.pyproject: # pragma: no cover
77 project = Path(self.pyproject).read_text()
79 if "[tool.typedal]" not in project and "[tool.pydal2sql]" in project:
80 # no typedal config, but existing p2s config:
81 return get_pydal2sql_config(self.pyproject)
83 return Config.load(
84 {
85 "db_type": self.dialect,
86 "format": "edwh-migrate",
87 "tables": self.tables,
88 "magic": self.magic,
89 "function": self.function,
90 "input": self.input,
91 "output": self.output,
92 "pyproject": self.pyproject,
93 }
94 )
96 def to_migrate(self) -> "MigrateConfig":
97 """
98 Convert the config to the format required by edwh-migrate.
99 """
100 from edwh_migrate import Config, get_config
102 if self.pyproject: # pragma: no cover
103 project = Path(self.pyproject).read_text()
105 if "[tool.typedal]" not in project and "[tool.migrate]" in project:
106 # no typedal config, but existing p2s config:
107 return get_config()
109 return Config.load(
110 {
111 "migrate_uri": self.database,
112 "schema_version": self.schema_version,
113 "redis_host": self.redis_host,
114 "migrate_cat_command": self.migrate_cat_command,
115 "database_to_restore": self.database_to_restore,
116 "migrate_table": self.migrate_table,
117 "flag_location": self.flag_location,
118 "create_flag_location": self.create_flag_location,
119 "schema": self.schema,
120 "db_folder": self.folder,
121 "migrations_file": self.output,
122 }
123 )
126def find_pyproject_toml(directory: str | None = None) -> typing.Optional[str]:
127 """
128 Find the project's config toml, looks up until it finds the project root (black's logic).
129 """
130 return black.files.find_pyproject_toml((directory or os.getcwd(),))
133def _load_toml(path: str | bool | None = True) -> tuple[str, dict[str, Any]]:
134 """
135 Path can be a file, a directory, a bool or None.
137 If it is True or None, the default logic is used.
138 If it is False, no data is loaded.
139 if it is a directory, the pyproject.toml will be searched there.
140 If it is a path, that file will be used.
141 """
142 if path is False:
143 toml_path = None
144 elif path in (True, None):
145 toml_path = find_pyproject_toml()
146 elif Path(str(path)).is_file():
147 toml_path = str(path)
148 else:
149 toml_path = find_pyproject_toml(str(path))
151 if not toml_path:
152 # nothing to load
153 return "", {}
155 try:
156 with open(toml_path, "rb") as f:
157 data = tomli.load(f)
159 return toml_path or "", typing.cast(dict[str, Any], data["tool"]["typedal"])
160 except Exception as e:
161 warnings.warn(f"Could not load typedal config toml: {e}", source=e)
162 return toml_path or "", {}
165def _load_dotenv(path: str | bool | None = True) -> tuple[str, dict[str, Any]]:
166 if path is False:
167 dotenv_path = None
168 elif path in (True, None):
169 dotenv_path = find_dotenv(usecwd=True)
170 elif Path(str(path)).is_file():
171 dotenv_path = str(path)
172 else:
173 dotenv_path = str(Path(str(path)) / ".env")
175 if not dotenv_path:
176 return "", {}
178 # 1. find everything with TYPEDAL_ prefix
179 # 2. remove that prefix
180 # 3. format values if possible
181 data = dotenv_values(dotenv_path)
182 data |= os.environ # higher prio than .env
184 typedal_data = {k.lower().removeprefix("typedal_"): v for k, v in data.items()}
186 return dotenv_path, typedal_data
189DB_ALIASES = {
190 "postgresql": "postgres",
191 "psql": "postgres",
192 "sqlite3": "sqlite",
193}
196def get_db_for_alias(db_name: str) -> str:
197 """
198 Convert a db dialect alias to the standard name.
199 """
200 return DB_ALIASES.get(db_name, db_name)
203DEFAULTS: dict[str, Any | typing.Callable[[dict[str, Any]], Any]] = {
204 "database": lambda data: data.get("db_uri") or "sqlite:memory",
205 "dialect": lambda data: get_db_for_alias(data["database"].split(":")[0])
206 if ":" in data["database"]
207 else data.get("db_type"),
208 "migrate": lambda data: not (data.get("input") or data.get("output")),
209 "folder": lambda data: data.get("db_folder"),
210 "flag_location": lambda data: f"{db_folder}/flags"
211 if (db_folder := (data.get("folder") or data.get("db_folder")))
212 else "/flags",
213 "pool_size": lambda data: 1 if data.get("dialect", "sqlite") == "sqlite" else 3,
214}
217def _fill_defaults(data: dict[str, Any], prop: str, fallback: Any = None) -> None:
218 default = DEFAULTS.get(prop, fallback)
219 if callable(default):
220 default = default(data)
221 data[prop] = default
224def fill_defaults(data: dict[str, Any], prop: str) -> None:
225 """
226 Fill missing property defaults with (calculated) sane defaults.
227 """
228 if data.get(prop, None) is None:
229 _fill_defaults(data, prop)
232TRANSFORMS: dict[str, typing.Callable[[dict[str, Any]], Any]] = {
233 "database": lambda data: data["database"]
234 if (":" in data["database"] or not data.get("dialect"))
235 else (data["dialect"] + "://" + data["database"])
236}
239def transform(data: dict[str, Any], prop: str) -> bool:
240 """
241 After the user has chosen a value, possibly transform it.
242 """
243 if fn := TRANSFORMS.get(prop):
244 data[prop] = fn(data)
245 return True
246 return False
249def expand_posix_vars(posix_expr: str, context: dict[str, str]) -> str:
250 """
251 Replace case-insensitive POSIX and Docker Compose-like environment variables in a string with their values.
253 Args:
254 posix_expr (str): The input string containing case-insensitive POSIX or Docker Compose-like variables.
255 context (dict): A dictionary containing variable names and their respective values.
257 Returns:
258 str: The string with replaced variable values.
260 See Also:
261 https://stackoverflow.com/questions/386934/how-to-evaluate-environment-variables-into-a-string-in-python
262 and ChatGPT
263 """
264 env = defaultdict(lambda: "")
265 for key, value in context.items():
266 env[key.lower()] = value
268 # Regular expression to match "${VAR:default}" pattern
269 pattern = r"\$\{([^}]+)\}"
271 def replace_var(match: re.Match[Any]) -> str:
272 var_with_default = match.group(1)
273 var_name, default_value = var_with_default.split(":") if ":" in var_with_default else (var_with_default, "")
274 return env.get(var_name.lower(), default_value)
276 return re.sub(pattern, replace_var, posix_expr)
279def expand_env_vars_into_toml_values(toml: dict[str, Any], env: dict[str, Any]) -> None:
280 """
281 Recursively expands POSIX/Docker Compose-like environment variables in a TOML dictionary.
283 This function traverses a TOML dictionary and expands POSIX/Docker Compose-like
284 environment variables (${VAR:default}) using values provided in the 'env' dictionary.
285 It performs in-place modification of the 'toml' dictionary.
287 Args:
288 toml (dict): A TOML dictionary with string values possibly containing environment variables.
289 env (dict): A dictionary containing environment variable names and their respective values.
291 Returns:
292 None: The function modifies the 'toml' dictionary in place.
294 Notes:
295 The function recursively traverses the 'toml' dictionary. If a value is a string or a list of strings,
296 it attempts to substitute any environment variables found within those strings using the 'env' dictionary.
298 Example:
299 toml_data = {
300 'key1': 'This has ${ENV_VAR:default}',
301 'key2': ['String with ${ANOTHER_VAR}', 'Another ${YET_ANOTHER_VAR}']
302 }
303 environment = {
304 'ENV_VAR': 'replaced_value',
305 'ANOTHER_VAR': 'value_1',
306 'YET_ANOTHER_VAR': 'value_2'
307 }
309 expand_env_vars_into_toml_values(toml_data, environment)
310 # 'toml_data' will be modified in place:
311 # {
312 # 'key1': 'This has replaced_value',
313 # 'key2': ['String with value_1', 'Another value_2']
314 # }
315 """
316 if not toml or not env:
317 return
319 for key, var in toml.items():
320 if isinstance(var, dict):
321 expand_env_vars_into_toml_values(var, env)
322 elif isinstance(var, list):
323 toml[key] = [expand_posix_vars(_, env) for _ in var if isinstance(_, str)]
324 elif isinstance(var, str):
325 toml[key] = expand_posix_vars(var, env)
326 else:
327 # nothing to substitute
328 continue
331def load_config(
332 _use_pyproject: bool | str | None = True, _use_env: bool | str | None = True, **fallback: Any
333) -> TypeDALConfig:
334 """
335 Combines multiple sources of config into one config instance.
336 """
337 # load toml data
338 # load .env data
339 # combine and fill with fallback values
340 # load typedal config or fail
341 toml_path, toml = _load_toml(_use_pyproject)
342 dotenv_path, dotenv = _load_dotenv(_use_env)
344 expand_env_vars_into_toml_values(toml, dotenv)
346 connection_name = dotenv.get("connection", "") or toml.get("default", "")
347 connection: dict[str, Any] = (toml.get(connection_name) if connection_name else toml) or {}
349 combined = connection | dotenv | fallback
350 combined = {k.replace("-", "_"): v for k, v in combined.items()}
352 combined["pyproject"] = toml_path
353 combined["connection"] = connection_name
355 for prop in TypeDALConfig.__annotations__:
356 fill_defaults(combined, prop)
358 for prop in TypeDALConfig.__annotations__:
359 transform(combined, prop)
361 return TypeDALConfig.load(combined, convert_types=True)