Coverage for src/typedal/config.py: 100%
132 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-02 16:17 +0200
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-02 16:17 +0200
1"""
2TypeDAL can be configured by a combination of pyproject.toml (static), env (dynamic) and code (programmic).
3"""
5import os
6import re
7import typing
8import warnings
9from collections import defaultdict
10from pathlib import Path
11from typing import Any, Optional
13import tomli
14from configuraptor import TypedConfig, alias
15from configuraptor.helpers import find_pyproject_toml
16from dotenv import dotenv_values, find_dotenv
18from .types import AnyDict
20if typing.TYPE_CHECKING: # pragma: no cover
21 from edwh_migrate import Config as MigrateConfig
22 from pydal2sql.typer_support import Config as P2SConfig
25class TypeDALConfig(TypedConfig):
26 """
27 Unified config for TypeDAL runtime behavior and migration utilities.
28 """
30 # typedal:
31 database: str
32 dialect: str
33 folder: str = "databases"
34 caching: bool = True
35 pool_size: int = 0
36 pyproject: str
37 connection: str = "default"
39 # pydal2sql:
40 input: str = ""
41 output: str = ""
42 noop: bool = False
43 magic: bool = True
44 tables: Optional[list[str]] = None
45 function: str = "define_tables"
47 # edwh-migrate:
48 # migrate uri = database
49 database_to_restore: Optional[str]
50 migrate_cat_command: Optional[str]
51 schema_version: Optional[str]
52 redis_host: Optional[str]
53 migrate_table: str = "typedal_implemented_features"
54 flag_location: str
55 create_flag_location: bool = True
56 schema: str = "public"
58 # typedal (depends on properties above)
59 migrate: bool = True
60 fake_migrate: bool = False
62 # aliases:
63 db_uri: str = alias("database")
64 db_type: str = alias("dialect")
65 db_folder: str = alias("folder")
67 # repr set by @beautify (by inheriting from TypedConfig)
69 def to_pydal2sql(self) -> "P2SConfig":
70 """
71 Convert the config to the format required by pydal2sql.
72 """
73 from pydal2sql.typer_support import Config, get_pydal2sql_config
75 if self.pyproject: # pragma: no cover
76 project = Path(self.pyproject).read_text()
78 if "[tool.typedal]" not in project and "[tool.pydal2sql]" in project:
79 # no typedal config, but existing p2s config:
80 return get_pydal2sql_config(self.pyproject)
82 return Config.load(
83 {
84 "db_type": self.dialect,
85 "format": "edwh-migrate",
86 "tables": self.tables,
87 "magic": self.magic,
88 "function": self.function,
89 "input": self.input,
90 "output": self.output,
91 "pyproject": self.pyproject,
92 }
93 )
95 def to_migrate(self) -> "MigrateConfig":
96 """
97 Convert the config to the format required by edwh-migrate.
98 """
99 from edwh_migrate import Config, get_config
101 if self.pyproject: # pragma: no cover
102 project = Path(self.pyproject).read_text()
104 if "[tool.typedal]" not in project and "[tool.migrate]" in project:
105 # no typedal config, but existing p2s config:
106 return get_config()
108 return Config.load(
109 {
110 "migrate_uri": self.database,
111 "schema_version": self.schema_version,
112 "redis_host": self.redis_host,
113 "migrate_cat_command": self.migrate_cat_command,
114 "database_to_restore": self.database_to_restore,
115 "migrate_table": self.migrate_table,
116 "flag_location": self.flag_location,
117 "create_flag_location": self.create_flag_location,
118 "schema": self.schema,
119 "db_folder": self.folder,
120 "migrations_file": self.output,
121 }
122 )
125def _load_toml(path: str | bool | Path | None = True) -> tuple[str, AnyDict]:
126 """
127 Path can be a file, a directory, a bool or None.
129 If it is True or None, the default logic is used.
130 If it is False, no data is loaded.
131 if it is a directory, the pyproject.toml will be searched there.
132 If it is a path, that file will be used.
133 """
134 if path is False:
135 toml_path = None
136 elif path in (True, None):
137 toml_path = find_pyproject_toml()
138 elif (_p := Path(str(path))) and _p.is_file():
139 toml_path = _p
140 else:
141 toml_path = find_pyproject_toml(str(path))
143 if not toml_path:
144 # nothing to load
145 return "", {}
147 try:
148 with open(toml_path, "rb") as f:
149 data = tomli.load(f)
151 return str(toml_path) or "", typing.cast(AnyDict, data["tool"]["typedal"])
152 except Exception as e:
153 warnings.warn(f"Could not load typedal config toml: {e}", source=e)
154 return str(toml_path) or "", {}
157def _load_dotenv(path: str | bool | None = True) -> tuple[str, AnyDict]:
158 fallback_data = {k.lower().removeprefix("typedal_"): v for k, v in os.environ.items()}
159 if path is False:
160 dotenv_path = None
161 fallback_data = {}
162 elif path in (True, None):
163 dotenv_path = find_dotenv(usecwd=True)
164 elif Path(str(path)).is_file():
165 dotenv_path = str(path)
166 else:
167 dotenv_path = str(Path(str(path)) / ".env")
169 if not dotenv_path:
170 return "", fallback_data
172 # 1. find everything with TYPEDAL_ prefix
173 # 2. remove that prefix
174 # 3. format values if possible
175 data = dotenv_values(dotenv_path)
176 data |= os.environ # higher prio than .env
178 typedal_data = {k.lower().removeprefix("typedal_"): v for k, v in data.items()}
180 return dotenv_path, typedal_data
183DB_ALIASES = {
184 "postgresql": "postgres",
185 "psql": "postgres",
186 "sqlite3": "sqlite",
187}
190def get_db_for_alias(db_name: str) -> str:
191 """
192 Convert a db dialect alias to the standard name.
193 """
194 return DB_ALIASES.get(db_name, db_name)
197DEFAULTS: dict[str, Any | typing.Callable[[AnyDict], Any]] = {
198 "database": lambda data: data.get("db_uri") or "sqlite:memory",
199 "dialect": lambda data: (
200 get_db_for_alias(data["database"].split(":")[0]) if ":" in data["database"] else data.get("db_type")
201 ),
202 "migrate": lambda data: not (data.get("input") or data.get("output")),
203 "folder": lambda data: data.get("db_folder"),
204 "flag_location": lambda data: (
205 f"{db_folder}/flags" if (db_folder := (data.get("folder") or data.get("db_folder"))) else "/flags"
206 ),
207 "pool_size": lambda data: 1 if data.get("dialect", "sqlite") == "sqlite" else 3,
208}
211def _fill_defaults(data: AnyDict, prop: str, fallback: Any = None) -> None:
212 default = DEFAULTS.get(prop, fallback)
213 if callable(default):
214 default = default(data)
215 data[prop] = default
218def fill_defaults(data: AnyDict, prop: str) -> None:
219 """
220 Fill missing property defaults with (calculated) sane defaults.
221 """
222 if data.get(prop, None) is None:
223 _fill_defaults(data, prop)
226TRANSFORMS: dict[str, typing.Callable[[AnyDict], Any]] = {
227 "database": lambda data: (
228 data["database"]
229 if (":" in data["database"] or not data.get("dialect"))
230 else (data["dialect"] + "://" + data["database"])
231 )
232}
235def transform(data: AnyDict, prop: str) -> bool:
236 """
237 After the user has chosen a value, possibly transform it.
238 """
239 if fn := TRANSFORMS.get(prop):
240 data[prop] = fn(data)
241 return True
242 return False
245def expand_posix_vars(posix_expr: str, context: dict[str, str]) -> str:
246 """
247 Replace case-insensitive POSIX and Docker Compose-like environment variables in a string with their values.
249 Args:
250 posix_expr (str): The input string containing case-insensitive POSIX or Docker Compose-like variables.
251 context (dict): A dictionary containing variable names and their respective values.
253 Returns:
254 str: The string with replaced variable values.
256 See Also:
257 https://stackoverflow.com/questions/386934/how-to-evaluate-environment-variables-into-a-string-in-python
258 and ChatGPT
259 """
260 env = defaultdict(lambda: "")
261 for key, value in context.items():
262 env[key.lower()] = value
264 # Regular expression to match "${VAR:default}" pattern
265 pattern = r"\$\{([^}]+)\}"
267 def replace_var(match: re.Match[Any]) -> str:
268 var_with_default = match.group(1)
269 var_name, default_value = var_with_default.split(":") if ":" in var_with_default else (var_with_default, "")
270 return env.get(var_name.lower(), default_value)
272 return re.sub(pattern, replace_var, posix_expr)
275def expand_env_vars_into_toml_values(toml: AnyDict, env: AnyDict) -> None:
276 """
277 Recursively expands POSIX/Docker Compose-like environment variables in a TOML dictionary.
279 This function traverses a TOML dictionary and expands POSIX/Docker Compose-like
280 environment variables (${VAR:default}) using values provided in the 'env' dictionary.
281 It performs in-place modification of the 'toml' dictionary.
283 Args:
284 toml (dict): A TOML dictionary with string values possibly containing environment variables.
285 env (dict): A dictionary containing environment variable names and their respective values.
287 Returns:
288 None: The function modifies the 'toml' dictionary in place.
290 Notes:
291 The function recursively traverses the 'toml' dictionary. If a value is a string or a list of strings,
292 it attempts to substitute any environment variables found within those strings using the 'env' dictionary.
294 Example:
295 toml_data = {
296 'key1': 'This has ${ENV_VAR:default}',
297 'key2': ['String with ${ANOTHER_VAR}', 'Another ${YET_ANOTHER_VAR}']
298 }
299 environment = {
300 'ENV_VAR': 'replaced_value',
301 'ANOTHER_VAR': 'value_1',
302 'YET_ANOTHER_VAR': 'value_2'
303 }
305 expand_env_vars_into_toml_values(toml_data, environment)
306 # 'toml_data' will be modified in place:
307 # {
308 # 'key1': 'This has replaced_value',
309 # 'key2': ['String with value_1', 'Another value_2']
310 # }
311 """
312 if not toml or not env:
313 return
315 for key, var in toml.items():
316 if isinstance(var, dict):
317 expand_env_vars_into_toml_values(var, env)
318 elif isinstance(var, list):
319 toml[key] = [expand_posix_vars(_, env) for _ in var if isinstance(_, str)]
320 elif isinstance(var, str):
321 toml[key] = expand_posix_vars(var, env)
322 else:
323 # nothing to substitute
324 continue
327def load_config(
328 connection_name: Optional[str] = None,
329 _use_pyproject: bool | str | None = True,
330 _use_env: bool | str | None = True,
331 **fallback: Any,
332) -> TypeDALConfig:
333 """
334 Combines multiple sources of config into one config instance.
335 """
336 # load toml data
337 # load .env data
338 # combine and fill with fallback values
339 # load typedal config or fail
340 toml_path, toml = _load_toml(_use_pyproject)
341 dotenv_path, dotenv = _load_dotenv(_use_env)
343 expand_env_vars_into_toml_values(toml, dotenv)
345 connection_name = connection_name or dotenv.get("connection", "") or toml.get("default", "")
346 connection: AnyDict = (toml.get(connection_name) if connection_name else toml) or {}
348 combined = connection | dotenv | fallback
349 combined = {k.replace("-", "_"): v for k, v in combined.items()}
351 combined["pyproject"] = toml_path
352 combined["connection"] = connection_name
354 for prop in TypeDALConfig.__annotations__:
355 fill_defaults(combined, prop)
357 for prop in TypeDALConfig.__annotations__:
358 transform(combined, prop)
360 return TypeDALConfig.load(combined, convert_types=True)