Coverage for src/edwh/tasks.py: 18%
1132 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-13 17:03 +0200
1import contextlib
2import datetime as dt
3import fnmatch
4import hashlib
5import io
6import json
7import os
8import pathlib
9import re
10import shlex
11import shutil
12import subprocess
13import sys
14import threading
15import time
16import tomllib
17import traceback
18import typing as t
19import warnings
20from collections import defaultdict
21from concurrent import futures
22from dataclasses import dataclass
23from getpass import getpass
24from pathlib import Path
25from typing import Optional
27import ewok
28import invoke
29import keyring
30import keyring.errors
31import tabulate
32import tomlkit # has more features than tomllib
33import yaml
34from dotenv import dotenv_values
35from ewok import Context, Task, format_frame, task
36from invoke import Promise, Runner
37from packaging.version import parse as parse_version
38from rapidfuzz import fuzz
39from ssh_agent_keyring.backend import SSHAgentKeyring
40from termcolor import colored, cprint
41from termcolor._types import Color
42from typing_extensions import Never
44from .__about__ import __version__ as edwh_version
45from .constants import (
46 DEFAULT_DOTENV_PATH,
47 DEFAULT_TOML_NAME,
48 DOCKER_COMPOSE,
49 FALLBACK_TOML_NAME,
50 FILE_START,
51 LEGACY_TOML_NAME,
52)
53from .discover import discover, get_hosts_for_service # noqa F401 - import for export (Remco afblijven)
54from .health import (
55 docker_inspect,
56 find_container_ids,
57 find_containers_ids,
58 get_healths,
59)
61# noinspection PyUnresolvedReferences
62# ^ keep imports for backwards compatibility (e.g. `from edwh.tasks import executes_correctly`)
63from .helpers import ( # noqa F401 - import for export
64 AnyDict,
65 ColorFn,
66 LineBufferHandler,
67 NoopHandler,
68 confirm,
69 dc_config,
70 dump_set_as_list,
71 executes_correctly,
72 execution_fails,
73 fabric_read,
74 fabric_write,
75 flatten,
76 interactive_selected_checkbox_values,
77 interactive_selected_radio_value,
78 noop,
79 parse_regex,
80 print_aligned,
81 rainbow,
82 run_pty,
83 run_pty_ok,
84 shorten,
85)
86from .helpers import generate_password as _generate_password
88# noinspection PyUnresolvedReferences
89# ^ keep imports for other tasks to register them!
90from .meta import is_installed, plugins, self_update # noqa
93def copy_fallback_toml(
94 tomlfile: str | Path = DEFAULT_TOML_NAME,
95 fallbacks: t.Collection[str | Path] = (LEGACY_TOML_NAME, FALLBACK_TOML_NAME),
96 force: bool = False,
97) -> bool:
98 tomlfile_path = Path(tomlfile)
100 if tomlfile_path.exists() and not force: 100 ↛ 101line 100 didn't jump to line 101 because the condition on line 100 was never true
101 return False
103 for fallback_name in fallbacks: 103 ↛ 112line 103 didn't jump to line 112 because the loop on line 103 didn't complete
104 fallback_path = Path(fallback_name)
106 if fallback_name is None or not fallback_path.exists():
107 continue
109 shutil.copy(fallback_path, tomlfile_path)
110 return True
112 tomlfile_path.touch()
113 return False
116def service_names(
117 service_arg: t.Collection[str] | None,
118 default: t.Literal["all", "minimal", "logs", "celeries"] | None = None,
119) -> list[str]:
120 """
121 Returns a list of matching servicenames based on ALL_SERVICES. filename globbing is applied.
123 Use service_names(['*celery*','pg*']) to select all celery services, and all of pg related instances.
124 :param service_arg: list of services or service selectors using wildcards
125 :param default: which services to return if service_arg is empty?
126 :return: list of unique services names that match the given list
127 """
129 config = TomlConfig.load()
130 if not config:
131 return []
133 selected = set()
134 if service_arg is None:
135 service_arg = []
136 elif isinstance(service_arg, str):
137 service_arg = service_arg.split(",")
138 else:
139 service_arg = list(flatten([_.split(",") for _ in service_arg]))
141 service_arg = [_.strip("/") for _ in service_arg] if service_arg else ([str(default)] if default else [])
143 # NOT elif because you can pass -s "minimal" -s "celeries" for example
144 if "all" in service_arg:
145 service_arg.remove("all")
146 service_arg.extend(config.all_services)
147 if "minimal" in service_arg:
148 service_arg.remove("minimal")
149 service_arg.extend(config.services_minimal)
150 if "logs" in service_arg:
151 service_arg.remove("logs")
152 service_arg.extend(config.services_log)
153 if "celeries" in service_arg:
154 service_arg.remove("celeries")
155 service_arg.extend(config.celeries)
156 if "pgq" in service_arg:
157 service_arg.remove("pgq")
158 service_arg.extend(config.pgq)
159 if "db" in service_arg and config.services_db:
160 service_arg.remove("db")
161 service_arg.extend(config.services_db)
163 # service_arg is specified, filter through all available services:
165 for service in service_arg:
166 selected.update(fnmatch.filter(config.all_services, service))
168 if service_arg and not selected:
169 # when no service matches the name, don't return an empty list, as that would `up` all services
170 # instead of the wanted list. This includes typos, where a single typo could cause all services to be started.
171 cprint(f"ERROR: No services found matching: {service_arg!r}", color="red")
172 exit(1)
173 return list(selected)
176def calculate_schema_hash(quiet: bool = False) -> str:
177 """
178 Calculates the sha1 digest of the files in the shared_code folder.
180 When anything is changed, it will have a different hash, so migrate will be triggered.
181 """
182 filenames = sorted(Path("./shared_code").glob("**/*"))
183 # ignore those pesky __pycache__ folders
184 filenames = [_ for _ in filenames if "__pycache__" not in str(_) and _.is_file()]
185 hasher = hashlib.sha256(b"")
186 for filename in filenames:
187 hasher.update(filename.read_bytes())
189 if not quiet:
190 print("schema hash: ", hasher.hexdigest())
191 return hasher.hexdigest()
194def task_for_namespace(ctx: Context, namespace: str, task_name: str) -> Task | None:
195 """
196 Get a task by namespace + task_name.
198 Example:
199 namespace: local, task_name: setup
200 """
202 if ns := ewok.find_namespace(ctx, namespace):
203 return t.cast(Task, ns.tasks.get(task_name))
205 return None
208def task_for_identifier(ctx: Context, identifier: str) -> Task | None:
209 collection = ewok.tasks(ctx)
211 return collection.tasks.get(identifier)
214def get_task(ctx: Context, identifier: str = "") -> Task | None:
215 """
216 Get a task by the identifier you would use in the terminal.
218 Example:
219 local.setup
220 """
221 if not identifier:
222 stack = traceback.extract_stack(limit=2)
223 cprint(
224 "WARN: get_task(identifier) is deprecated in favor of get_task(invoke.Context, identifier)",
225 color="yellow",
226 )
227 format_frame(stack[0])
228 return None
230 if "." in identifier:
231 return task_for_namespace(ctx, *identifier.split("."))
232 else:
233 return task_for_identifier(ctx, identifier)
236_dotenv_settings: dict[str, dict[str, str]] = {}
239def _apply_env_vars_to_template(source_lines: list[str], env: dict[str, str]) -> list[str]:
240 needle = re.compile(r"# *template:")
242 new_lines = []
243 for line in source_lines:
244 if not needle.findall(line):
245 # nothing found, try next line
246 new_lines.append(line)
247 continue
249 # split on template definition:
250 old, template = needle.split(line)
251 template = template.strip()
252 # save the indention part, add an addition if no indention was found
253 indention = (re.findall(r"^\s*", old) + [""])[0] # noqa: RUF005 would make this complex
254 if not old.lstrip().startswith("#"):
255 # skip comment only lines
256 new = template.format(**env)
257 # reconstruct the line for the yaml file
258 line = f"{indention}{new} # template: {template}"
259 new_lines.append(line)
260 return new_lines
263# used for treafik config
264def apply_dotenv_vars_to_yaml_templates(yaml_path: Path, dotenv_path: Path = DEFAULT_DOTENV_PATH) -> None:
265 """Indention preserving templating of yaml files, uses dotenv_path for variables.
267 Pythong formatting is used with a dictionary of environment variables used from os environment variables
268 updated by the dotenv_path parsed .dotenv entries.
269 Templating is found using `# template:`
270 indention is saved, everything after the above indicator is python string formatted and written back.
272 Example:
273 |config:
274 | email: some@one.com # template: {EMAIL}
276 assuming dotenv file contains:
277 |EMAIL=yep@thatsme.com
279 applying this function will result in:
280 |config:
281 | email: yep@thatsme.com # template: {EMAIL}
282 """
283 env = os.environ.copy()
284 env |= read_dotenv(dotenv_path)
285 # env_variable_re = re.compile(r'\$[A-Z0-9]')
286 with yaml_path.open(mode="r+") as yaml_file:
287 source_lines = yaml_file.read().split("\n")
288 new_lines = _apply_env_vars_to_template(source_lines, env)
289 # move filepointer to the start of the file
290 yaml_file.seek(0, FILE_START)
291 # write all lines and newlines to the file
292 yaml_file.write("\n".join(new_lines))
293 # and remove any part that might be left over (when the new file is shorter than the old one)
294 yaml_file.truncate()
297# Singleton but depending on 'fname' (toml file name) and 'dotenv_path'
298tomlconfig_singletons: dict[tuple[str, str], "TomlConfig"] = {}
301def throw(error: Exception) -> Never:
302 """
303 Functional raise, useful for if ... else ... or callbacks.
304 """
305 raise error
308class ServicesTomlConfig(t.TypedDict, total=False):
309 """
310 [services] section of .toml
311 """
313 services: t.Literal["discover"] | list[str]
314 minimal: list[str]
315 include_celeries_in_minimal: str # 'true'/'1' or 'false'/'0'
316 include_pgq_in_minimal: str # 'true'/'1' or 'false'/'0'
317 log: list[str]
318 db: list[str]
321# todo: keyof<ServicesTomlConfig> or something?
322TomlKeys = t.Literal["services", "minimal", "include_celeries_in_minimal", "include_pgq_in_minimal", "log", "db"]
325class ConfigTomlDict(t.TypedDict, total=True):
326 """
327 Data from .toml
328 """
330 services: ServicesTomlConfig
331 dotenv: AnyDict
334def boolish(value: t.Literal["y", "yes", "t", "true", "1", "n", "no", "false", "f", "0"] | str | int) -> bool:
335 """
336 Convert a given value to a boolean.
338 Args:
339 value (Union[str, int]): The value to be converted.
340 Accepts strings representing true/false values such as "y", "yes", "t", "true", "1" for true
341 and "n", "no", "false", "f", "0" for false, as well as integers.
343 Returns:
344 bool: The boolean representation of the input value.
345 """
346 return bool(value) and str(value)[0].strip().lower() in {"y", "t", "1"}
349@dataclass
350class TomlConfig:
351 config: ConfigTomlDict
352 all_services: list[str]
353 celeries: list[str]
354 pgq: list[str]
355 services_minimal: list[str]
356 services_log: list[str]
357 services_db: list[str]
358 services_health: list[str]
360 dotenv_path: Path
362 # __loaded was replaced with tomlconfig_singletons
364 @classmethod
365 def load(
366 cls,
367 fname: str | Path = DEFAULT_TOML_NAME,
368 dotenv_path: Optional[Path] = None,
369 cache: bool = True,
370 ) -> "TomlConfig | None":
371 """
372 Load config toml file, raising an error if it does not exist.
374 Since this file should be in .git error suppression is not needed.
375 Returns a dictionary with CONFIG, ALL_SERVICES, CELERIES and MINIMAL_SERVICES
376 """
377 singleton_key = (str(fname), str(dotenv_path))
378 ctx = t.cast(Context, invoke.Context())
380 if cache and (instance := tomlconfig_singletons.get(singleton_key)): 380 ↛ 381line 380 didn't jump to line 381 because the condition on line 380 was never true
381 return instance
383 config_path = Path(fname) # probably config.toml
384 dc_path = Path("docker-compose.yml")
386 if not dc_path.exists(): 386 ↛ 387line 386 didn't jump to line 387 because the condition on line 386 was never true
387 cprint(
388 "docker-compose.yml file is missing, toml config could not be loaded. Functionality may be limited.",
389 color="yellow",
390 )
391 return None
393 if not config_path.exists(): 393 ↛ 396line 393 didn't jump to line 396 because the condition on line 393 was always true
394 setup(ctx)
396 config = read_toml_config(config_path)
397 # todo: if setup runs, reload config
399 if "services" not in config: 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true
400 setup(ctx)
401 config = read_toml_config(config_path)
403 toml_keys = t.get_args(TomlKeys)
404 for toml_key in toml_keys:
405 if toml_key not in config["services"]: 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true
406 setup(ctx)
407 config = read_toml_config(config_path)
409 if config["services"].get("services", "discover") == "discover": 409 ↛ 410line 409 didn't jump to line 410 because the condition on line 409 was never true
410 compose = load_dockercompose_with_includes(dc_path=dc_path)
412 all_services = list(compose["services"].keys())
413 else:
414 all_services = t.cast(list[str], config["services"]["services"])
416 celeries = [s for s in all_services if "celery" in s.lower()]
417 pgq = [s for s in all_services if "pgq" in s.lower()]
419 minimal_services = config["services"]["minimal"]
420 if boolish(config["services"].get("include_celeries_in_minimal", "false")): 420 ↛ 421line 420 didn't jump to line 421 because the condition on line 420 was never true
421 minimal_services += celeries
422 if boolish(config["services"].get("include_pgq_in_minimal", "false")): 422 ↛ 423line 422 didn't jump to line 423 because the condition on line 422 was never true
423 minimal_services += pgq
425 tomlconfig_singletons[singleton_key] = instance = TomlConfig(
426 config=config,
427 all_services=all_services,
428 celeries=celeries,
429 pgq=pgq,
430 services_minimal=minimal_services,
431 services_log=config["services"]["log"],
432 services_db=config["services"]["db"],
433 services_health=config["services"].get("health", []),
434 dotenv_path=Path(config.get("dotenv", {}).get("path", dotenv_path or DEFAULT_DOTENV_PATH)),
435 )
436 return instance
439def process_env_file(env_path: Path) -> dict[str, str]:
440 if not env_path.exists():
441 return {}
443 values = dotenv_values(env_path)
444 return t.cast(dict[str, str], dict(values))
447def exists_nonempty(path: Path) -> bool:
448 """
449 Checks whether a given file path exists and is non-empty.
451 This function determines if the specified path represents an existing
452 file and whether it contains any data (i.e., its size is greater than 0).
454 Args:
455 path (Path): A Path object representing the file path to check.
457 Returns:
458 bool: True if the file exists and is non-empty, otherwise False.
459 """
460 return path.exists() and path.stat().st_size > 0
463def read_dotenv(env_path: Path = DEFAULT_DOTENV_PATH) -> dict[str, str]:
464 """
465 Read .env file from env_path and return a dict of key/value pairs.
467 If the .env file doesn't exist at env_path, traverse up the directory tree
468 looking for one, stopping when a docker-compose.* file is found (project root).
470 :param env_path: optional path to .env file
471 :return: dict of key/value pairs from the .env file
472 """
473 if not env_path:
474 # for backwards compatibility, if None is passed: still use the default.
475 env_path = DEFAULT_DOTENV_PATH
477 cache_key = str(env_path) if env_path else "."
478 if existing := _dotenv_settings.get(cache_key):
479 # 'cache'
480 return existing
482 # First try the exact path provided
483 if exists_nonempty(env_path):
484 items = process_env_file(env_path)
485 _dotenv_settings[cache_key] = items
486 return items
488 # If not found and it's the default name, traverse up the tree
489 if env_path.name == DEFAULT_DOTENV_PATH.name:
490 current_dir = Path.cwd()
492 while current_dir != current_dir.parent: # Stop at filesystem root
493 # Look for .env in current directory
494 potential_env = current_dir / DEFAULT_DOTENV_PATH.name
495 if exists_nonempty(potential_env):
496 items = process_env_file(potential_env)
497 _dotenv_settings[cache_key] = items
498 return items
500 # Check if we've reached a project root (docker-compose file exists)
501 if any(current_dir.glob("docker-compose.*")) and current_dir != Path.cwd():
502 # Found project root, stop searching if we're not in the original directory
503 break
505 # Move up one directory
506 current_dir = current_dir.parent
508 # If still not found, return empty dict (existing behavior)
509 items = {}
510 _dotenv_settings[cache_key] = items
511 return items
514# noinspection PyDefaultArgument
515def warn_once(
516 warning: str,
517 previously_shown: list[str] = [],
518 color: Optional[Color] = None,
519 **print_kwargs: t.Any,
520) -> None:
521 """
522 Mutable default 'previously_shown' is there on purpose, to track which warnings were already shown!
523 """
524 if warning in previously_shown:
525 # already seen
526 return
528 previously_shown.append(warning)
530 cprint(
531 warning,
532 color=color,
533 **print_kwargs,
534 )
537type DefaultFn = t.Callable[[], Optional[str]]
540def check_env(
541 key: str,
542 default: Optional[str] | DefaultFn,
543 comment: str,
544 # optionals:
545 prefix: Optional[str] = None,
546 suffix: Optional[str] = None,
547 # note: 'postfix' should be 'suffix' but to be backwards compatible we can't just remove it!
548 postfix: Optional[str] = None,
549 # different config paths:
550 env_path: Optional[str | Path] = None,
551 force_default: Optional[bool] = False,
552 allowed_values: t.Iterable[str] = (),
553 toml_path: None = None,
554) -> str:
555 """
556 Test if key is in .env file path, appends prompted or default value if missing.
558 Args:
559 key: The environment variable key to check.
560 default: The default value to use if the key is not found. Can also be a function for lazy evaluation.
561 comment: A comment describing the purpose of the environment variable.
562 prefix: An optional prefix to prepend to the key.
563 suffix: An optional suffix to append to the key.
564 postfix: An optional parameter for backward compatibility with 'suffix'.
565 env_path: An optional path to the environment file.
566 force_default: Whether to force the default value even if the key exists.
567 allowed_values: A list of allowed values for the environment variable.
568 toml_path: Optional path to a TOML configuration file.
570 Returns:
571 The value of the environment variable, either from the file, default, or forced.
574 Example:
575 > check_env(
576 > key="SOME_KEY",
577 > default=lambda c: slow_function()
578 > comment="This key has a lazily evaluated default",
579 > ...
580 > )
581 """
582 if toml_path:
583 warn_once(f"Deprecated: toml_path ({toml_path} is not used by check_env anymore.)", color="yellow")
585 env_path = Path(env_path or DEFAULT_DOTENV_PATH)
586 if not env_path.exists():
587 env_path.parent.mkdir(parents=True, exist_ok=True)
588 env_path.touch()
590 # config = TomlConfig.load(toml_path, env_path)
591 env = read_dotenv(env_path)
593 if key in env:
594 return env[key]
596 if suffix and postfix:
597 warnings.warn(
598 "! both a 'suffix' and a 'postfix' parameter were specified, "
599 "but only 'suffix' will be used since 'postfix' is just an alias!",
600 category=DeprecationWarning,
601 )
602 elif postfix:
603 warnings.warn(
604 "The 'postfix' option has been replaced by 'suffix' and may be removed in the future.",
605 category=DeprecationWarning,
606 )
608 suffix = suffix or postfix
610 if callable(default):
611 default = default() # type: ignore
613 non_interactive = os.environ.get("EDWH_NON_INTERACTIVE", "0") == "1"
614 from_env = os.environ.get("EDWH_FROM_ENV", "0") == "1"
616 if force_default:
617 value = default or ""
618 elif from_env:
619 value = os.environ.get(key, default or "")
620 if not value:
621 raise RuntimeError(f"Environment variable {key} not found and no default provided (--from-env mode)")
622 elif non_interactive:
623 raise RuntimeError(f"No default value provided for {key} in --non-interactive mode")
624 else:
625 response = input(f"Enter value for {key} ({comment})\n default=`{default}`: ")
626 value = response.strip() or default or ""
628 if allowed_values and value not in allowed_values:
629 raise ValueError(f"Invalid value '{response}'. Please choose one of {allowed_values}")
631 str_value = str(value)
633 if prefix:
634 str_value = prefix + str_value
635 if suffix:
636 str_value += suffix
638 with env_path.open(mode="a") as env_file:
639 # append mode ensures we're writing at the end
640 env_file.write(f"\n{key.upper()}={str_value}")
642 # update in memory too:
643 env[key] = str_value
645 return str_value
648def get_env_value(key: str, default: str | type[Exception] = KeyError) -> str:
649 """
650 Get a specific env value by name.
651 If no default is given and the key is not found, a KeyError is raised.
652 """
653 env = read_dotenv()
654 if key in env:
655 return env[key]
656 elif isinstance(default, type) and issubclass(default, Exception):
657 raise default(key)
659 return default
662def set_env_value(path: Path, target: str, value: str | None) -> None:
663 """
664 Update/set environment variables in the .env file, keeping comments intact.
666 set_env_value(Path('.env'), 'SCHEMA_VERSION', schemaversion)
668 Args:
669 path: pathlib.Path designating the .env file
670 target: key to write, probably best to use UPPERCASE
671 value: string value to write (or anything that converts to a string using str()).
672 If None, the key will be removed from the file (if present) and not added.
673 """
674 path.touch(exist_ok=True)
676 with path.open(mode="r") as env_file:
677 # open the .env file and read every line in the inlines
678 inlines = env_file.read().split("\n")
680 outlines = [] # lines for output
681 geschreven = False
682 for line in inlines:
683 if line.strip().startswith("#"):
684 # ignore comments
685 outlines.append(line)
686 continue
687 # remove redundant whitespace
688 line = line.strip()
689 if not line:
690 # remove empty lines
691 continue
692 # convert to tuples
693 key, _oldvalue = line.split("=", 1)
694 # clean the key and value
695 key = key.strip()
696 if key == target:
697 if value is None:
698 # Remove this key by not adding it to outlines
699 geschreven = True
700 continue
701 # add the new tuple to the lines
702 outlines.append(f"{key}={value}")
703 geschreven = True
704 else:
705 # or leave it as it is
706 outlines.append(line)
708 if not geschreven and value is not None:
709 outlines.append(f"{target.strip().upper()}={value.strip()}")
711 with path.open(mode="w") as env_file:
712 env_file.write("\n".join(outlines))
713 env_file.write("\n")
716def write_content_to_toml_file(
717 content_key: TomlKeys,
718 content: str | list[str] | None,
719 filename: str | Path = DEFAULT_TOML_NAME,
720 allow_empty: bool = False,
721) -> None:
722 if not (content or allow_empty):
723 return
725 filepath = Path(filename)
727 config_toml_file = read_toml_config(filepath)
728 config_toml_file["services"][content_key] = content # type: ignore
730 write_toml_config(filepath, config_toml_file)
733def get_content_from_toml_file(
734 services: list[str],
735 toml_contents: ConfigTomlDict,
736 content_key: TomlKeys,
737 content: str,
738 default: list[str] | str,
739 overwrite: bool = False,
740 allow_empty: bool = False,
741) -> list[str] | str | None:
742 """
743 Gets content from a TOML file.
744 feat/ew_setup_2084/gwen
745 :param services: A list of services.
746 :param toml_contents: A dictionary representing the TOML file.
747 :param content_key: The key to look for in the TOML file.
748 :param content: The content to display to the user.
749 :param default: The default value to return if the conditions are not met.
750 :param overwrite: don't skip if key already exists
751 :param allow_empty: add an option to the dropdown to select no containers (e.g. for a service without database)
753 :return: The content from the TOML file or the default value.
754 :rtype: Any
755 """
757 has_existing_value = "services" in toml_contents and content_key in toml_contents["services"]
759 if has_existing_value and not overwrite:
760 print("skipping", content_key)
761 return ""
763 selected: set[str] = set()
764 if has_existing_value:
765 selected.update(toml_contents["services"][content_key])
766 elif default:
767 selected.update(default)
769 selection = interactive_selected_checkbox_values(services, content, selected=selected, allow_empty=allow_empty)
770 if allow_empty and selection is None:
771 return None
773 return selection or default
776def setup_config_file(filename: str | Path = DEFAULT_TOML_NAME) -> None:
777 """
778 sets up config.toml for use
779 """
780 filepath = Path(filename)
782 config_toml_file = tomlkit.loads(filepath.read_text())
783 if "services" not in config_toml_file:
784 filepath.write_text("\n[services]\n")
787def read_toml_config(fp: Path) -> ConfigTomlDict:
788 """
789 Read the config at filepath, and cast to the right typeddict.
790 """
791 config_toml_file = tomlkit.loads(fp.read_text())
793 return t.cast(ConfigTomlDict, config_toml_file)
796def write_toml_config(fp: Path, config: ConfigTomlDict) -> int:
797 return fp.write_text(tomlkit.dumps(config))
800def include_services(
801 service: str, services: list[str], key: TomlKeys, config_toml_file: ConfigTomlDict, overwrite: bool
802):
803 # adds to services
804 if not services:
805 write_content_to_toml_file(key, "false")
806 elif services and ("services" not in config_toml_file or key not in config_toml_file["services"] or overwrite):
807 # check if user wants to include service
808 include_service = (
809 "true" if confirm(f"do you want to include {service} in minimal [Yn]: ", default=True) else "false"
810 )
811 write_content_to_toml_file(key, include_service)
814def write_user_input_to_config_toml(
815 all_services: list[str],
816 filename: str | Path = DEFAULT_TOML_NAME,
817 overwrite: bool = False,
818) -> TomlConfig | None:
819 """
820 write chosen user dockers to config.toml
822 :param all_services: list of all docker services that are in the docker-compose.yml
823 :param filename: which toml file to write to (default = .toml)
824 :param overwrite: by default, skip keys that already have a value
825 :return:
826 """
827 filepath = Path(filename)
828 services_no_workers = [service for service in all_services if "pgq" not in service and "celery" not in service]
829 services_pgq = [service for service in all_services if "pgq" in service]
830 services_celery = [service for service in all_services if "celery" in service]
831 setup_config_file()
833 # services
834 services_list = "discover"
835 write_content_to_toml_file("services", services_list)
837 config_toml_file = read_toml_config(filepath)
839 include_services("pgq", services_pgq, "include_pgq_in_minimal", config_toml_file, overwrite)
840 include_services("celery", services_celery, "include_celeries_in_minimal", config_toml_file, overwrite)
842 # get chosen services for minimal and logs
843 minimal_services = (
844 services_no_workers
845 if config_toml_file["services"]["services"] == "discover"
846 else config_toml_file["services"]["services"]
847 )
849 # services
850 content = get_content_from_toml_file(
851 minimal_services,
852 config_toml_file,
853 "minimal",
854 "select minimal services you want to run on `ew up`: ",
855 [],
856 overwrite=overwrite,
857 )
858 write_content_to_toml_file("minimal", content, filename)
860 content = get_content_from_toml_file(
861 minimal_services,
862 config_toml_file,
863 "log",
864 "select services to be logged: ",
865 [],
866 overwrite=overwrite,
867 )
868 write_content_to_toml_file("log", content, filename)
870 # db
871 possibly_postgres = [_ for _ in minimal_services if "pg-" in _]
872 content = get_content_from_toml_file(
873 minimal_services,
874 config_toml_file,
875 "db",
876 "select database containers: ",
877 possibly_postgres,
878 overwrite=overwrite,
879 allow_empty=True,
880 )
881 write_content_to_toml_file("db", content or [], filename, allow_empty=content is None)
883 return TomlConfig.load(filename, cache=False)
886def load_dockercompose_with_includes(
887 c: Optional[invoke.Context] = None,
888 dc_path: str | Path = "docker-compose.yml",
889) -> AnyDict:
890 """
891 Since we're using `docker compose` with includes, simply yaml loading docker-compose.yml is not enough anymore.
893 This function uses the `docker compose config` command to properly load the entire config with all enabled services.
894 """
895 if not c: 895 ↛ 896line 895 didn't jump to line 896 because the condition on line 895 was never true
896 c = t.cast(Context, invoke.Context())
898 if not isinstance(dc_path, Path): 898 ↛ 901line 898 didn't jump to line 901 because the condition on line 898 was always true
899 dc_path = Path(dc_path)
901 if not dc_path.exists(): 901 ↛ 902line 901 didn't jump to line 902 because the condition on line 901 was never true
902 raise FileNotFoundError(dc_path)
904 if ran := c.run(f"{DOCKER_COMPOSE} -f {dc_path} config", hide=True): 904 ↛ anywhereline 904 didn't jump anywhere: it always raised an exception.
905 processed_config = ran.stdout.strip()
906 # mimic a file to load the yaml from
907 fake_file = io.StringIO(processed_config)
908 return t.cast(AnyDict, yaml.safe_load(fake_file))
909 else:
910 return {}
913def prompt_validate_sudo_pass(c: Context):
914 sudo_pass = getpass("Please enter the sudo password: ")
915 c.config.sudo.password = sudo_pass
917 try:
918 result = c.sudo("echo ''", warn=True, hide=True)
919 if not (result and result.ok):
920 raise invoke.exceptions.AuthFailure(result, "sudo")
922 cprint("Sudo password accepted!", color="green", file=sys.stderr)
923 return sudo_pass
924 except invoke.exceptions.AuthFailure as e:
925 cprint(str(e), color="red", file=sys.stderr)
926 return None
929def ssh_agent_keyring_config_path() -> Path:
930 config_root = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config"))
931 return config_root / "ssh-agent-keyring" / "config.json"
934def use_configured_ssh_agent_keyring() -> bool:
935 if not ssh_agent_keyring_config_path().exists():
936 return False
938 keyring.set_keyring(SSHAgentKeyring())
939 return True
942def configure_ssh_agent_keyring() -> bool:
943 result = subprocess.run(["ssh-add", "-L"], text=True, capture_output=True, check=False)
944 public_keys = [key for key in result.stdout.splitlines() if key.startswith("ssh-")]
945 if not public_keys:
946 cprint("No public keys are available from SSH_AUTH_SOCK.", color="red")
947 return False
949 options = {
950 public_key: f"{parts[0]} {parts[1][:20]}… {' '.join(parts[2:])}".strip()
951 for public_key in public_keys
952 if len(parts := public_key.split()) >= 2
953 }
954 public_key = interactive_selected_radio_value(options, prompt="Select the SSH key for EDWH's encrypted keyring:")
955 if not public_key:
956 return False
958 config_path = ssh_agent_keyring_config_path()
959 config_path.parent.mkdir(parents=True, exist_ok=True)
960 config = json.loads(config_path.read_text()) if config_path.exists() else {}
961 config["public_key"] = public_key
962 config_path.write_text(json.dumps(config, indent=2) + "\n")
964 keyring.set_keyring(SSHAgentKeyring())
965 return True
968def ensure_keyring_unlocked() -> bool:
969 """Offer a secure SSH-agent fallback when the system keyring is locked."""
970 use_configured_ssh_agent_keyring()
972 try:
973 keyring.get_password("edwh", "sudo")
974 except keyring.errors.KeyringLocked:
975 if confirm("The system keyring is locked. Would you like to use your SSH agent instead? [Yn]", default=True):
976 return configure_ssh_agent_keyring()
977 return False
979 return True
982@task()
983def require_sudo(c: Context) -> bool:
984 """
985 Can be used as a 'pre' hook for invoke tasks to make sure sudo is ready to be used,
986 without prompting for a password later on (which could fail due to not passing data to stdin on a remote host).
988 Usage:
989 @task(pre=[require_sudo])
990 def setup(c): ...
992 # or, if you're not in a @task but you do have access to c (Context), e.g. in a helper function:
993 def my_func(c):
994 if require_sudo(c):
995 c.sudo('echo "I am the captain now."')
997 """
998 use_configured_ssh_agent_keyring()
1000 ran = c.run("sudo --non-interactive echo ''", warn=True, hide=True)
1001 if ran and ran.ok:
1002 # prima
1003 return True
1005 with contextlib.suppress(Exception):
1006 if current := keyring.get_password("edwh", "sudo"):
1007 c.config.sudo.password = current
1008 return True
1010 if prompt_validate_sudo_pass(c):
1011 return True
1012 else:
1013 cprint("Stopping now.")
1014 exit(1)
1017def build_toml(c: Context, overwrite: bool = False) -> TomlConfig | None:
1018 try:
1019 docker_compose = load_dockercompose_with_includes(c)
1020 except FileNotFoundError:
1021 cprint("docker-compose.yml file is missing, setup could not be completed!", color="red")
1022 return None
1024 services: AnyDict = docker_compose["services"]
1025 return write_user_input_to_config_toml(list(services.keys()), overwrite=overwrite)
1028@task()
1029def sudo(c: Context):
1030 if not ensure_keyring_unlocked():
1031 return
1033 # 1.
1034 # check current status in keyring
1035 try:
1036 current = keyring.get_password("edwh", "sudo")
1037 except Exception:
1038 current = None
1040 # 2. change text based on current status (re-authorize)
1041 if current:
1042 allow = confirm(
1043 "Would you like to re-authorize edwh to run sudo commands without password entry? [Yn]",
1044 default=True,
1045 )
1046 else:
1047 allow = confirm(
1048 "Would you like to authorize edwh to run sudo commands without password entry? [yN]",
1049 default=False,
1050 )
1052 if allow:
1053 # if yes: add to keyring
1054 if sudo_pass := prompt_validate_sudo_pass(c):
1055 keyring.set_password("edwh", "sudo", sudo_pass)
1056 else:
1057 exit(1)
1059 else:
1060 # else: remove from keyring
1061 keyring.delete_password("edwh", "sudo")
1064@task(
1065 pre=[require_sudo],
1066 help={
1067 "new_config_toml": "Remove existing config.toml and create a fresh one",
1068 "from_env": "Read configuration values from environment variables instead of prompting "
1069 "(forces non-interactive mode)",
1070 "non_interactive": "Skip all interactive prompts; fail if required configuration values are missing",
1071 },
1072 hookable=True,
1073)
1074def setup(
1075 c: Context,
1076 new_config_toml: bool = False,
1077 _retry: bool = False,
1078 from_env: bool = False,
1079 non_interactive: bool = False,
1080) -> dict:
1081 """
1082 Sets up config.toml and tries to run setup in local tasks.py if it exists
1083 """
1084 config_toml = Path(DEFAULT_TOML_NAME)
1085 dc_path = Path("docker-compose.yml")
1087 if from_env: 1087 ↛ 1088line 1087 didn't jump to line 1088 because the condition on line 1087 was never true
1088 os.environ["EDWH_NON_INTERACTIVE"] = "1"
1089 os.environ["EDWH_FROM_ENV"] = "1"
1090 elif non_interactive: 1090 ↛ 1091line 1090 didn't jump to line 1091 because the condition on line 1090 was never true
1091 os.environ["EDWH_NON_INTERACTIVE"] = "1"
1093 if ( 1093 ↛ 1101line 1093 didn't jump to line 1101 because the condition on line 1093 was never true
1094 new_config_toml
1095 and config_toml.exists()
1096 and confirm(
1097 colored(f"Are you sure you want to remove the {DEFAULT_TOML_NAME}? [yN]", "red"),
1098 default=False,
1099 )
1100 ):
1101 config_toml.unlink()
1103 copy_fallback_toml(force=False) # only if .toml is missing, try to copy default.toml
1105 if dc_path.exists(): 1105 ↛ 1118line 1105 didn't jump to line 1118 because the condition on line 1105 was always true
1106 print("getting services...")
1108 try:
1109 # run `docker compose config` to build a yaml with all processing done, include statements included.
1110 build_toml(c)
1111 except Exception as e:
1112 cprint(
1113 f"Something went wrong trying to create a {DEFAULT_TOML_NAME} from docker-compose.yml ({e})",
1114 color="red",
1115 )
1116 # this could be because 'include' requires a variable that's setup in local task, so still run that:
1117 else:
1118 cprint("docker-compose file is missing, setup might not be completed properly!", color="yellow")
1120 # local/plugin setup happens here because of `hookable`
1121 return {}
1124@task()
1125def search_adjacent_setting(c: Context, key: str, silent: bool = False) -> AnyDict:
1126 """
1127 Search for key in all ../*/.env files.
1128 """
1129 key = key.upper()
1130 if not silent:
1131 print("search for ", key)
1132 envs = (pathlib.Path(c.cwd) / "..").glob("*/.env")
1133 adjacent_settings = {}
1134 for env_path in envs:
1135 value = read_dotenv(env_path).get(key)
1136 project = env_path.parent.name
1137 if not silent:
1138 print(f"{project:>20} : {value}")
1139 adjacent_settings[project] = value
1140 return adjacent_settings
1143def next_value(c: Context, key: list[str] | str, lowest: int, silent: bool = True) -> int:
1144 """Find all other project settings using key, adding 1 to max of all values, or defaults to lowest.
1146 next_value(c, 'REDIS_PORT', 6379) -> might result 6379, or 6381 if this is the third project to be initialised
1147 next_value(c, ['PGPOOL_PORT','POSTGRES_PORT','PGBOUNCER_PORT'], 5432) -> finds the next port searching for all keys.
1148 """
1149 keys = [key] if isinstance(key, str) else key
1150 all_settings: AnyDict = {}
1151 for key in keys:
1152 settings = search_adjacent_setting(c, key, silent)
1153 all_settings |= {f"{k}/{key}": v for k, v in settings.items() if v}
1154 if not silent:
1155 print()
1156 values = {int(v) for v in all_settings.values() if v}
1157 return max(values) + 1 if any(values) else lowest
1160THREE_WEEKS = 60 * 24 * 7 * 3
1163@task()
1164def clean_old_sessions(c: Context, relative_glob: str = "web2py/apps/*/sessions", minutes: int = THREE_WEEKS) -> None:
1165 for directory in Path.cwd().glob(relative_glob):
1166 c.sudo(f'find "{directory}" -type f -mmin +{minutes} -exec rm -f "{{}}" +;')
1167 remove_empty_dirs(c, directory)
1170@task()
1171def remove_empty_dirs(c: Context, path: str | Path) -> None:
1172 c.sudo(f'find "{path}" -type d -exec rmdir --ignore-fail-on-non-empty {{}} +')
1175def set_permissions(
1176 c: Context,
1177 path: str,
1178 uid: int = 1050,
1179 gid: int = 1050,
1180 filepermissions: int = 664,
1181 directorypermissions: int = 775,
1182) -> None:
1183 """
1184 Set all directories in path to 'directorypermissions',
1185 all files to 'filepermissions'
1186 and chown the right user+group.
1187 """
1188 # sudo(f'find "{path}" -type d -print0 | sudo xargs --no-run-if-empty -0 chmod {directorypermissions}')
1189 c.sudo(f'find "{path}" -type d -exec chmod {directorypermissions} {{}} +')
1190 # find all files, print the output, feed those to xargs which converts lines in to arguments to the chmod command.
1191 # sudo(f'find "{path}" -type f -print0 | sudo xargs --no-run-if-empty -0 chmod {filepermissions}')
1192 c.sudo(f'find "{path}" -type f -exec chmod {filepermissions} {{}} +')
1193 # simply apply new ownership to each and every directory
1194 c.sudo(f'chown -R {uid}:{gid} "{path}" ')
1197@task(help=dict(silent="do not echo the password"))
1198def generate_password(_: Context, silent: bool = False, dice: int = 6) -> str:
1199 """
1200 Generate a diceware password using --dice 6.
1202 Arggs:
1203 _: invoke Context
1204 silent: don't print the generated password?
1205 dice: amount of words to generate
1207 """
1208 return _generate_password(silent=silent, dice=dice)
1211def fuzzy_match(val1: str, val2: str, verbose: bool = False) -> float:
1212 """
1213 Get the similarity score between two values.
1215 Used by `edwh settings -f ...` when no exact match was found.
1216 """
1217 similarity = fuzz.partial_ratio(val1, val2)
1218 if verbose:
1219 print(f"similarity of {val1} and {val2} is {similarity}", file=sys.stderr)
1220 return similarity
1223def _settings(find: t.Optional[str], fuzz_threshold: int = 75) -> t.Iterable[tuple[str, t.Any]]:
1224 all_settings = read_dotenv().items()
1225 if find is None:
1226 # don't loop
1227 return all_settings
1228 else:
1229 find = find.upper()
1230 # if nothing found exactly, try again but fuzzy (could be slower)
1231 exact_match = [(k, v) for k, v in all_settings if find in k.upper() or find in v.upper()]
1232 return exact_match or [(k, v) for k, v in all_settings if fuzzy_match(k.upper(), find) > fuzz_threshold]
1235# noinspection PyUnusedLocal
1236@task(
1237 help=dict(find="search for this specific setting", as_json="output as json dictionary"),
1238 flags={
1239 "as_json": ("j", "json", "as-json"),
1240 "fuzz_threshold": ("t", "fuzz-threshold"),
1241 },
1242)
1243def settings(_: Context, find: Optional[str] = None, fuzz_threshold: int = 75, as_json: bool = False) -> None:
1244 """
1245 Show all settings in .env file or search for a specific setting using -f/--find.
1246 """
1247 rows = _settings(find, fuzz_threshold)
1248 if as_json:
1249 print(json.dumps(dict(rows), indent=3))
1250 else:
1251 print(tabulate.tabulate(rows, headers=["Setting", "Value"]))
1254def show_related_settings(ctx: Context, services: list[str]) -> None:
1255 config = dc_config(ctx)
1257 rows: AnyDict = {}
1258 for service in services:
1259 if service_settings := _settings(service):
1260 rows |= service_settings
1261 else:
1262 with contextlib.suppress(TypeError, KeyError):
1263 rows |= config["services"][service]["environment"]
1265 print(tabulate.tabulate(rows.items(), headers=["Setting", "Value"]))
1268@task(aliases=("volume",))
1269def volumes(ctx: Context) -> None:
1270 """
1271 Show container and volume names.
1273 Based on `docker-compose ps -q` ids and `docker inspect` output.
1274 """
1275 lines: list[AnyDict] = []
1276 ran = ctx.run(f"{DOCKER_COMPOSE} ps -q", hide=True, warn=True)
1277 stdout = ran.stdout if ran else ""
1278 for container_id in stdout.strip().split("\n"):
1279 with contextlib.suppress(EnvironmentError):
1280 docker_info = docker_inspect(ctx, container_id)
1281 if not isinstance(docker_info, list):
1282 continue
1284 info = docker_info[0]
1285 container = info["Name"]
1286 lines.extend(
1287 dict(container=container, volume=volume)
1288 for volume in [_["Name"] for _ in info["Mounts"] if _["Type"] == "volume"]
1289 )
1291 print(tabulate.tabulate(lines, headers="keys"))
1294def check_paused(ctx: Context, service: str) -> bool:
1295 """Check if a service container is paused."""
1296 result = ctx.run(f"{DOCKER_COMPOSE} ps --format json {service}", hide=True, warn=True)
1297 try:
1298 container_info = json.loads(result.stdout.strip())
1299 if isinstance(container_info, list):
1300 container_info = container_info[0] if container_info else {}
1302 state = container_info.get("State", "")
1303 return state == "paused"
1304 except (json.JSONDecodeError, KeyError, IndexError):
1305 return False
1308def get_service_dependencies(ctx: Context, service: str) -> list[str]:
1309 """Get the dependencies (depends_on) for a service from docker-compose config."""
1310 result = ctx.run(f"{DOCKER_COMPOSE} config --format json", hide=True, warn=True)
1311 try:
1312 config = json.loads(result.stdout.strip())
1313 services = config.get("services", {})
1314 service_config = services.get(service, {})
1315 depends_on: dict[str, t.Any] = service_config.get("depends_on", {})
1317 # depends_on can be a list or a dict
1318 if isinstance(depends_on, dict):
1319 return list(depends_on.keys())
1320 elif isinstance(depends_on, list):
1321 return depends_on
1322 return []
1323 except (json.JSONDecodeError, KeyError) as e:
1324 print(f"Could not get dependencies for {service}: {e}")
1325 return []
1328def get_paused_services_with_deps(ctx: Context, services: list[str]) -> list[str]:
1329 """Get all paused services including their dependencies.
1331 Args:
1332 ctx: Invoke context
1333 services: List of service names to check
1335 Returns:
1336 List of paused service names (including dependencies)
1337 """
1338 # Collect all services including dependencies
1339 all_services = set(services)
1340 for service in services:
1341 dependencies = get_service_dependencies(ctx, service)
1342 all_services.update(dependencies)
1344 # Check which services are paused
1345 return [svc for svc in all_services if check_paused(ctx, svc)]
1348# noinspection PyShadowingNames
1351@task(
1352 help=dict(
1353 service="Service to up, defaults to .toml's [services].minimal. Can be used multiple times, handles wildcards.",
1354 build="request a build be performed first",
1355 quickest="restart only, no down;up",
1356 stop_timeout="timeout for stopping services, defaults to 2 seconds",
1357 tail="tails the log of restart services, defaults to False",
1358 clean="adds `--renew-anon-volumes --build` to `docker-compose up` command ",
1359 ),
1360 iterable=["service"],
1361 flags={
1362 "tail": ("tail", "logs", "l"), # instead of -a; NOTE: 'tail' must be first (matches parameter name)
1363 },
1364 hookable=True,
1365)
1366def up(
1367 ctx: Context,
1368 service: t.Collection[str] | None = None,
1369 no_build: bool = False,
1370 quickest: bool = False,
1371 stop_timeout: int = 2,
1372 tail: bool = False,
1373 clean: bool = False,
1374 show_settings: bool = True,
1375 wait: bool = False,
1376) -> dict:
1377 """Restart (or down;up) some or all services, after an optional rebuild."""
1378 config = TomlConfig.load()
1379 # recalculate the hash and save it, so with the next up, migrate will see differences and start migration
1380 set_env_value(DEFAULT_DOTENV_PATH, "SCHEMA_VERSION", calculate_schema_hash())
1381 # test for --service arguments, if none given: use defaults
1382 services = service_names(service or (config.services_minimal if config else []))
1383 services_ls = " ".join(services)
1385 # Check for paused containers and unpause them
1386 if paused_services := get_paused_services_with_deps(ctx, services):
1387 paused_ls = " ".join(paused_services)
1388 cprint(f"Unpausing and stopping services: {paused_ls}", "blue")
1389 ctx.run(f"{DOCKER_COMPOSE} unpause {paused_ls}", pty=True)
1390 # unpaused containers often get unhealthy so also stop them:
1391 ctx.run(f"{DOCKER_COMPOSE} stop {paused_ls}", pty=True)
1393 if quickest:
1394 ctx.run(f"{DOCKER_COMPOSE} restart {services_ls}")
1395 else:
1396 ctx.run(f"{DOCKER_COMPOSE} stop -t {stop_timeout} {services_ls}")
1397 # note: checking if build is required due to outdated versions seems undoable, docker has no api for it
1398 # so we're just adding --build. There also is no --pull, so you need to run ew build or dc pull manually
1400 ctx.run(
1401 f"{DOCKER_COMPOSE} up "
1402 f"{'--renew-anon-volumes' if clean else ''} "
1403 f"{'' if no_build else '--build'} "
1404 f"-d {services_ls}",
1405 pty=True,
1406 )
1408 if show_settings:
1409 show_related_settings(ctx, services)
1410 if tail:
1411 ctx.run(f"{DOCKER_COMPOSE} logs --tail=10 -f {services_ls}")
1412 if wait:
1413 health(ctx, services, wait=True)
1415 # local/plugin up happens here because of `hookable`
1416 return {
1417 "services": services,
1418 }
1421@task(aliases=("health-inspect",))
1422def inspect_health(ctx, container: str, quiet: bool = False) -> dict:
1423 tab = " " * 2
1424 result = {}
1426 with contextlib.suppress(OSError):
1427 container_ids = find_container_ids(ctx, container) or [container]
1429 for container_id in container_ids:
1430 result[container_id] = docker_inspect(ctx, container_id, '--format "{{json .State.Health }}"')
1432 if result and not quiet:
1433 print(tab + yaml.dump(result, allow_unicode=True).replace("\n", f"\n{tab}"))
1435 return result
1438@task(
1439 iterable=("service",),
1440)
1441def wait_until_healthy(ctx: Context, services: t.Iterable[str] = (), quiet: bool = False):
1442 initial_length = 0
1443 # for every container with a health check, wait for it to be either healthy or dead (not starting)
1444 while missing := [_.container for _ in get_healths(ctx, *services) if _ and _.health == "starting"]:
1445 if not quiet:
1446 msg = f" Waiting for {missing}" + " " * 25
1447 if not initial_length:
1448 initial_length = len(msg)
1450 print(msg, end="\r")
1452 # wait is done, now print empty line to cleanup print traces:
1453 if initial_length and not quiet:
1454 print(" " * initial_length)
1456 return 0
1459@task(
1460 flags={
1461 "show_all": ("all", "a"),
1462 },
1463 iterable=("service",),
1464)
1465def health(
1466 ctx: Context,
1467 service: t.Collection[str] | None = None,
1468 wait: bool = False,
1469 show_all: bool = False,
1470 quiet: bool = False,
1471 verbose: bool = False,
1472) -> int:
1473 """
1474 Show health status for docker containers
1476 Args:
1477 ctx: invoke context
1478 service: which services to show logs for.
1479 If you have a 'health' section in your .toml, those services will be used by default.
1480 Otherwise, 'all' will be used by default.
1481 wait: should the command wait until all services are healthy? Defaults to only showing status once and exiting.
1482 show_all: show all services. Alias for `-s all`
1483 quiet: don't print anything, only return amount of unhealthy containers
1484 verbose: print health inspection for unhealthy containers
1486 Returns:
1487 Number of unhealthy services (0 is good, just like bash exit codes).
1488 Should always be 0 if you use --wait
1489 """
1490 config = TomlConfig.load()
1491 # test for --service arguments, if none given: use defaults
1492 if show_all:
1493 services = service_names("all")
1494 elif service:
1495 services = service_names(service)
1496 elif config:
1497 services = service_names(config.services_health or config.all_services)
1498 else:
1499 services = []
1501 if wait:
1502 return wait_until_healthy(ctx, services, quiet=quiet)
1504 healths = get_healths(ctx, *services)
1505 if not quiet:
1506 for health_status in sorted((_ for _ in healths if _ is not None), key=lambda h: (h.level, h.container)):
1507 print(f"- {health_status}")
1508 if verbose and not health_status.ok and health_status.container_id:
1509 inspect_health(ctx, health_status.container_id)
1511 # return amount of sick containers:
1512 return sum(not _.ok for _ in healths)
1515class DockerProject(t.TypedDict):
1516 count: int
1517 container_statuses: list[str]
1520@task(aliases=("psa",))
1521def ps_all(ctx):
1522 """
1523 Show Docker Compose projects with container counts and summarized status.
1524 """
1525 result = ctx.run("docker ps --format '{{json .}}'", hide=True)
1526 lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
1528 projects: dict[str, DockerProject] = defaultdict(lambda: {"count": 0, "container_statuses": []})
1530 for line in lines:
1531 container_data = json.loads(line)
1532 container_name = container_data["Names"]
1533 container_state = container_data.get("State", "")
1534 container_status_text = container_data.get("Status", "").lower()
1536 project_name = container_name.split("-")[0]
1537 projects[project_name]["count"] += 1
1539 # Determine container status with priority: unhealthy > paused > healthy/ok > others
1540 if "unhealthy" in container_status_text or "health: starting" in container_status_text:
1541 container_status = "unhealthy"
1542 elif "paused" in container_state:
1543 container_status = "paused"
1544 elif "healthy" in container_status_text:
1545 container_status = "healthy"
1546 elif container_state == "running":
1547 container_status = "ok"
1548 else:
1549 container_status = container_state
1551 projects[project_name]["container_statuses"].append(container_status)
1553 table_rows = []
1554 for project_name, info in sorted(projects.items()):
1555 if not isinstance(info, dict):
1556 continue
1558 statuses_set = set(info["container_statuses"])
1560 if "unhealthy" in statuses_set:
1561 project_status = "unhealthy"
1562 elif "paused" in statuses_set:
1563 project_status = "paused"
1564 # The <= operator for sets checks if statuses_set is a subset of {"healthy", "ok"}
1565 # i.e., all containers are either healthy or ok
1566 elif statuses_set <= {"healthy", "ok"}:
1567 project_status = "ok"
1568 elif len(statuses_set) > 1:
1569 project_status = "mixed"
1570 else:
1571 project_status = next(iter(statuses_set))
1573 table_rows.append((project_name, info["count"], project_status))
1575 print(tabulate.tabulate(table_rows, headers=["Project", "Containers", "Status"], tablefmt="pipe"))
1578@task(
1579 iterable=["service", "columns"],
1580 help=dict(
1581 service="Service to query, can be used multiple times, handles wildcards.",
1582 quiet="Only show container ids. Useful for scripting.",
1583 columns="Which columns to display?",
1584 full="Don't truncate the command.",
1585 ),
1586 flags={
1587 "show_all": ("all", "a"),
1588 },
1589)
1590def ps(
1591 ctx: Context,
1592 quiet: bool = False,
1593 service: t.Collection[str] | None = None,
1594 columns: t.Collection[str] | None = None,
1595 full: bool = False,
1596 show_all: bool = False,
1597) -> None:
1598 """
1599 Show process status of services.
1600 """
1601 trunc_after = 30
1602 if not Path("docker-compose.yml").exists():
1603 cprint("You're not in a docker compose environment.", color="red")
1604 if confirm("Would you like to see all running environments? [Yn]", default=True):
1605 ps_all(ctx)
1606 return
1608 flags = []
1610 if show_all:
1611 flags.append("-a")
1612 if quiet:
1613 flags.append("-q")
1615 # we may trunc it ourselves:
1616 flags.append("--no-trunc")
1618 flags.extend(service_names(service or []))
1620 args_str = " ".join(flags)
1622 ran = ctx.run(
1623 f"{DOCKER_COMPOSE} ps --format json {args_str}",
1624 warn=True,
1625 hide=True,
1626 )
1627 ps_output = ran.stdout.strip() if ran else ""
1629 services = []
1631 # list because it's ordered
1632 selected_columns = list(columns or []) or ["Name", "Command", "Image", "State", "Health", "Ports"]
1634 for service_json in ps_output.split("\n"):
1635 if not service_json:
1636 # empty line
1637 continue
1639 service_dict = json.loads(service_json)
1640 service_dict = {k: v for k, v in service_dict.items() if k in selected_columns}
1641 if not full:
1642 service_dict["Command"] = shorten(service_dict["Command"], trunc_after)
1643 service_dict["Image"] = shorten(service_dict["Image"], trunc_after)
1645 service_dict = dict(sorted(service_dict.items(), key=lambda x: selected_columns.index(x[0])))
1646 services.append(service_dict)
1648 print(tabulate.tabulate(services, headers="keys"))
1651@task(
1652 help=dict(
1653 quiet="Only show ids (mostly directories). Useful for scripting.",
1654 ),
1655)
1656def ls(ctx: Context, quiet: bool = False) -> None:
1657 """
1658 List running compose projects.
1659 """
1660 ctx.run(f"{DOCKER_COMPOSE} ls {'-q' if quiet else ''}")
1663def get_docker_info(ctx: Context, services: list[str]) -> dict[str, AnyDict]:
1664 """
1665 Return a dict of {id: service}
1666 """
1667 # -aq doesn't keep the same order of services, so use json format to get ID with service name.
1668 # use --no-trunc to get full ID instead of short one
1669 if ran := ctx.run(f"{DOCKER_COMPOSE} ps --format json --no-trunc -a {' '.join(services)}", hide=True):
1670 rows = ran.stdout
1671 else:
1672 rows = ""
1674 result = {}
1676 for line in rows.split("\n"):
1677 if not line:
1678 continue
1680 # each line contains one json object
1681 info = json.loads(line)
1683 result[info["ID"]] = info
1685 return result
1688T_Stream = t.Literal["stdout", "stderr", "out", "err", ""]
1691def follow_logs(
1692 ctx: Context,
1693 container_id: str,
1694 project: str,
1695 longest_name: int,
1696 color: ColorFn,
1697 since: str | None,
1698 verbose: bool,
1699 timestamps: bool,
1700 stream: T_Stream = "",
1701 filter_pattern: str = "",
1702 stop_event: threading.Event | None = None,
1703) -> bool:
1704 """
1705 Follows logs of a specified Docker container while optionally filtering and formatting output.
1707 This function actively monitors the logs of a given container, allowing for custom filtering
1708 through a regular expression pattern and handling streams like `stdout` or `stderr`. It is
1709 designed to handle specific options such as including timestamps or verbose prefixes, and it
1710 can restart the log retrieval process in case of certain interruptions.
1712 Parameters:
1713 ctx (Context): Execution context used to run commands.
1714 container_id (str): ID of the container whose logs are being followed.
1715 project (str): Project or prefix name associated with the container.
1716 longest_name (int): Length of the longest name used for alignment in output.
1717 color (ColorFn): Function to apply color formatting to output, typically the container name.
1718 since (str | None): Initial timestamp or date from which logs should be retrieved, if available.
1719 verbose (bool): Whether to include verbose prefixes (e.g., stream type) in output.
1720 timestamps (bool): Whether to include timestamps from the logs in the output.
1721 stream (Literal["stdout", "stderr", "out", "err", ""]): Stream type to handle in the output.
1722 An empty string ("") implies both streams will be followed.
1723 filter_pattern (str): Regular expression pattern used to filter log entries. Defaults to an
1724 empty string, meaning no filtering is applied.
1726 Returns:
1727 bool: True if the log following process terminates validly, False otherwise.
1729 Raises:
1730 None explicitly defined, but will handle `KeyboardInterrupt` gracefully during execution.
1731 """
1732 if stream not in t.get_args(T_Stream):
1733 raise ValueError(f"Invalid stream value: '{stream}'.")
1735 # Get container name for prefix
1736 name_result = ctx.run(
1737 "docker inspect --format='{{.Name}}' %(container)s" % {"container": container_id},
1738 hide=True,
1739 warn=True,
1740 )
1742 container_name = name_result.stdout.strip().lstrip("/")
1743 container_name = container_name.removeprefix(f"{project}-").ljust(longest_name + 3, " ")
1745 prefix = color(f"{container_name} | ")
1747 re_filter_fn = parse_regex(filter_pattern) if filter_pattern else None
1749 stdout_handler = (
1750 LineBufferHandler(f"{prefix}out | " if verbose else prefix, sys.stdout, filter_fn=re_filter_fn)
1751 if stream in ("out", "stdout", "")
1752 else NoopHandler()
1753 )
1754 stderr_handler = (
1755 LineBufferHandler(f"{prefix}err | " if verbose else prefix, sys.stderr, filter_fn=re_filter_fn)
1756 if stream in ("out", "stdout", "")
1757 else NoopHandler()
1758 )
1760 # Loop until container state is 'exited'
1761 process: Optional[Promise] = None
1762 runner: Optional[Runner] = None
1763 while True:
1764 try:
1765 # Check container state
1766 result = ctx.run(
1767 "docker inspect --format='{{.State.Status}}' %(container)s" % {"container": container_id},
1768 hide=True,
1769 warn=True,
1770 )
1772 if result.failed:
1773 # machine is dead
1774 return False
1776 # Follow logs with timestamps, starting from last_timestamp if available
1777 args = ["docker", "logs", "--follow", container_id]
1778 if timestamps:
1779 args.append("--timestamps")
1781 # If we have a last timestamp, use it to start from where we left off
1782 if since:
1783 args.extend(("--since", since))
1785 # Run the docker logs command with our watcher
1786 cmd = shlex.join(args)
1787 process = ctx.run(cmd, pty=True, warn=True, asynchronous=True)
1788 runner = process.runner
1790 try:
1791 while not runner.process_is_finished:
1792 # start loop - check for stop event
1793 if stop_event and stop_event.is_set():
1794 return True
1796 while runner.stdout:
1797 stdout_handler.process(runner.stdout.pop(0))
1799 while runner.stderr:
1800 stderr_handler.process(runner.stderr.pop(0))
1802 time.sleep(0.1)
1803 except ChildProcessError:
1804 # --since <datetime> includes rows at that datetime so `dt.timedelta(microseconds=1)` is added:
1805 since = (dt.datetime.now() + dt.timedelta(microseconds=1)).isoformat()
1806 time.sleep(0.1)
1807 continue
1809 except KeyboardInterrupt:
1810 # Cancel the process if it's still running
1811 if runner and not runner.process_is_finished:
1812 runner.stop()
1813 runner.kill()
1814 return True
1816 # idk how we got here
1817 return False
1820@task(
1821 aliases=("log",),
1822 iterable=["service"],
1823 flags={
1824 "show_all": ("all", "a"),
1825 "filter_pattern": ("filter", "p"), # -p for pattern, -f is already for follow
1826 },
1827 help={
1828 "service": "What services to follow. "
1829 "Defaults to services in the `log` section of `.toml`, can be applied multiple times. ",
1830 "show_all": "Ignore --service and show all service logs (same as `-s '*'`).",
1831 "follow": "Keep scrolling with the output (default, use --no-follow or --limit <n> or --sort to disable).",
1832 "timestamps": "Add timestamps (on by default, use --no-timestamps to disable)",
1833 "limit": "Start with how many lines of history, don't follow.",
1834 "sort": "Sort the output by timestamp: forced timestamp and mutual exclusive with follow.",
1835 "since": "Filter by age (2024-05-03T12:00:00, 1 hour, now); in UTC",
1836 "new": "Don't show old entries (conflicts with since, same as --since now)",
1837 "stream": "Filter by stdout/stderr (defaults to both), only used when following",
1838 "filter_pattern": "Search by term or regex, only used when following",
1839 "verbose": "Show slightly more info, like full timestamps.",
1840 },
1841)
1842def logs(
1843 ctx: Context,
1844 service: t.Collection[str] | None = None,
1845 follow: bool = True,
1846 limit: Optional[int] = None,
1847 sort: bool = False,
1848 show_all: bool = False,
1849 verbose: bool = False,
1850 timestamps: bool = True,
1851 since: Optional[str] = None,
1852 new: bool = False,
1853 stream: T_Stream = "",
1854 filter_pattern: str = "",
1855) -> list[bool]:
1856 """Smart docker logging"""
1858 if new and since:
1859 raise ValueError("Cannot use --new and --since together")
1860 if new:
1861 since = "1s"
1863 if sort and follow:
1864 raise ValueError("--sort is mutually exclusive with following logs")
1866 services = service_names([], default="all") if show_all else service_names(service or [], default="logs")
1868 if limit or not follow or sort:
1869 if filter_pattern:
1870 raise ValueError("--filter is exclusive with --limit, --no-follow and --sort")
1872 # use basic logs
1873 cmdline = [f"{DOCKER_COMPOSE} logs", f"--tail={limit or 500}"]
1874 cmdline.extend(services)
1875 if sort or timestamps:
1876 # add timestamps
1877 cmdline.append("-t")
1879 if sort:
1880 cmdline.append(r'| sed -E "s/^([^|]*)\|([^Z]*Z)(.*)$/\2|\1|\3/" | sort')
1882 if since:
1883 cmdline.extend(["--since", since])
1885 return [ctx.run(" ".join(cmdline), echo=verbose, pty=True).ok]
1887 # else use fancy logs
1889 # now find containers for these services:
1890 # -> `py4web` can map to `py4web-1, py4web-2` etc
1891 promises: list[futures.Future[bool]] = []
1892 colors = rainbow()
1893 containers = get_docker_info(ctx, services)
1895 if not containers:
1896 cprint(f"No running containers found for services {services}", color="red")
1897 exit(1)
1898 elif len(containers) != len(services):
1899 cprint("Amount of requested services does not match the amount of running containers!", color="yellow")
1901 # for adjusting the | location
1902 longest_name = max([len(_["Service"]) for _ in containers.values()])
1904 with futures.ThreadPoolExecutor() as executor:
1905 stop_event = threading.Event()
1907 for service in services:
1908 for container_id in ctx.run(f"{DOCKER_COMPOSE} ps -aq {service}", hide=True).stdout.split("\n"):
1909 if not (container_info := containers.get(container_id)):
1910 # empty or whitespace only
1911 continue
1913 future = executor.submit(
1914 follow_logs,
1915 ctx,
1916 container_id,
1917 container_info["Project"],
1918 longest_name,
1919 next(colors),
1920 since,
1921 verbose,
1922 timestamps,
1923 stream,
1924 filter_pattern,
1925 stop_event,
1926 )
1927 promises.append(future)
1929 # Wait for all futures to complete
1930 # This mimics the original join_all behavior to return the list of results
1931 try:
1932 return [promise.result() for promise in promises]
1933 except KeyboardInterrupt:
1934 print("Ctrl-C pressed, stopping log threads...")
1935 stop_event.set()
1936 return []
1939def start_logs(c: Context, service: t.Collection[str] | None = None, args: str = ""):
1940 """
1941 Normal edwh logs can't just be called with `edwh.tasks.logs` so this wrapper makes it easier.
1943 (otherwise `logs` will try to elevate permissions and get confused due to not being called directly)
1944 """
1945 service = service_names(service or ())
1947 args = " ".join(f"-s {s}" for s in service) + f" {args}"
1949 return c.run(f"edwh logs {args}", pty=True)
1952@task(
1953 iterable=["service"],
1954 help=dict(service="Service to stop, can be used multiple times, handles wildcards."),
1955 hookable=True,
1956)
1957def stop(ctx: Context, service: t.Collection[str] | None = None) -> None:
1958 """
1959 Stops services using docker-compose stop.
1960 """
1961 service = service_names(service or [])
1962 ctx.run(f"{DOCKER_COMPOSE} stop {' '.join(service)}")
1965@task(
1966 iterable=["service"],
1967 help=dict(service="Service to stop, can be used multiple times, handles wildcards."),
1968 hookable=True,
1969)
1970def down(ctx: Context, service: t.Collection[str] | None = None) -> None:
1971 """
1972 Stops services using docker-compose down.
1973 """
1974 service = service_names(service or []) if service else []
1976 ctx.run(f"{DOCKER_COMPOSE} down {' '.join(service)}", pty=True)
1979@task(
1980 iterable=["service"],
1981)
1982def restart(c: Context, service: t.Collection[str] | None = None, quiet: bool = False, force: bool = False):
1983 """
1984 Restart Docker services by sending termination signals (ctrl-c/SIGINT; if force: SIGKILL, SIGTERM).
1986 Defaults to 'py4web' if no services specified.
1987 Shows logs unless 'quiet' flag is enabled.
1989 Arguments:
1990 c (Context): The execution context.
1991 service (Collection[str], optional): A collection of service names to restart. Defaults to restarting
1992 the 'py4web' service.
1993 quiet (bool): A flag indicating whether to suppress logs display after restarting services.
1994 force: send SIGKILL + SIGTERM instead of SIGINT
1996 Raises:
1997 Executes a system command to restart the desired services.
1998 Should be used within an environment that supports thisfunctionality.
2000 Returns:
2001 None
2002 """
2003 service = service_names(service or ["py4web"])
2005 service_filter = "|".join(service)
2006 kill = "kill -15 1 || kill -9 1" if force else "kill -2 1"
2007 command = (
2008 'docker ps --filter "name=%(name)s" --format "{{.Names}}" | '
2009 'xargs -I {} docker exec {} sh -c "%(kill)s"' % dict(name=service_filter, kill=kill)
2010 )
2012 c.run(command)
2014 if not quiet:
2015 start_logs(c, service)
2018@task(
2019 hookable=True,
2020)
2021def upgrade(ctx: Context, build: bool = False) -> None:
2022 if build:
2023 ctx.run(f"{DOCKER_COMPOSE} build")
2024 else:
2025 ctx.run(f"{DOCKER_COMPOSE} pull")
2026 stop(ctx)
2027 ctx.run(f"{DOCKER_COMPOSE} up -d")
2030@task(
2031 help=dict(
2032 yes="Don't ask for confirmation, just do it. "
2033 "(unless requirements.in files are found and the `edwh-pipcompile-plugin` is not installed)",
2034 skip_compile="Skip the compilation of requirements.in files to requirements.txt files (e.g. for PRD).",
2035 ),
2036 hookable=True,
2037)
2038def build(ctx: Context, yes: bool = False, skip_compile: bool = False, pull: bool = True) -> None:
2039 """
2040 Build all services.
2042 Will test for the presence of `edwh-pipcompile-plugin` and use it to compile
2043 requirements.in files to requirements.txt files in child directories.
2044 """
2045 # Path.cwd() uses absolute paths, Path() is the same but relative
2046 reqs = list(Path().rglob("*/*.in"))
2048 if pip_compile := get_task(ctx, "pip.compile"):
2049 with_compile = not skip_compile
2050 else:
2051 with_compile = False
2053 cprint("`edwh-pipcompile-plugin` not found, unable to compile requirements.in files.", "red")
2054 cprint("💡 Install with `edwh plugin.add pipcompile`", "blue")
2055 print()
2056 print("Possible files to compile:")
2057 for req in reqs:
2058 print(" * ", req)
2060 if not (state_of_development := get_env_value("STATE_OF_DEVELOPMENT", "")):
2061 cprint("Warning: No SOD found. Add STATE_OF_DEVELOPMENT to the .env file", "yellow")
2063 is_dev = state_of_development == "ONT"
2065 if not reqs:
2066 cprint("No .in files found to compile!", "yellow")
2067 elif with_compile and pip_compile is not None and is_dev:
2068 for idx, req in enumerate(reqs, 1):
2069 reqtxt = req.parent / "requirements.txt"
2070 cprint(
2071 f"{idx}/{len(reqs)}: working on {req}",
2072 "blue",
2073 )
2074 missing = not reqtxt.exists()
2075 outdated = not missing and reqtxt.stat().st_ctime < req.stat().st_ctime
2077 if missing or outdated:
2078 print("The .txt file is outdated." if outdated else "requirements.txt doesn't exist.")
2080 question = f"compile {req}? [Yn]"
2081 if outdated:
2082 question = f"re{question}" # recompile
2084 if yes or confirm(question, default=True):
2085 pip_compile(ctx, str(req.parent))
2086 else:
2087 print("still current")
2088 else:
2089 print("Compilation of requirements.in files skipped.")
2091 print()
2092 prompt = "Pull and build docker images? [yN]" if pull else "Build docker images? [yN]"
2094 if yes or is_dev or confirm(prompt, default=False):
2095 if pull:
2096 ctx.run(f"{DOCKER_COMPOSE} pull --ignore-buildable", pty=True)
2098 ctx.run(f"{DOCKER_COMPOSE} build", pty=True, env=dict(COMPOSE_BAKE="true"))
2101@task(
2102 help=dict(
2103 service="Service to rebuild, can be used multiple times, handles wildcards.",
2104 force_rebuild="uses --no-cache option for docker-compose build",
2105 ),
2106 iterable=["service"],
2107)
2108def rebuild(
2109 ctx: Context,
2110 service: t.Collection[str] | None = None,
2111 force_rebuild: bool = False,
2112) -> None:
2113 """
2114 Downs ALL services, then rebuilds services using docker-compose build.
2115 """
2116 ctx.run(f"{DOCKER_COMPOSE} down")
2117 services = service_names(service)
2119 cache_flag = "--no-cache" if force_rebuild else ""
2120 services_str = " ".join(services)
2121 ctx.run(f"{DOCKER_COMPOSE} build {cache_flag} {services_str}")
2124@task()
2125def docs(ctx: Context, reinstall: bool = False) -> bool:
2126 """
2127 Local hosted mkdocs documentation.
2129 Installs mkdocs if unavailable.
2130 """
2131 if reinstall:
2132 print("Installing mkdocs and dependencies...")
2133 ok = True
2134 ctx.run("pipx uninstall mkdocs", hide=True, warn=True)
2136 ran = ctx.run("pipx install mkdocs", hide=True, warn=True)
2137 ok &= bool(ran and ran.ok)
2138 ran = ctx.run(
2139 "pipx inject mkdocs mkdocs-material plantuml-markdown",
2140 hide=True,
2141 warn=True,
2142 )
2143 ok &= bool(ran and ran.ok)
2144 print("result:", ok)
2145 return ok
2146 else:
2147 ran = ctx.run("mkdocs serve", warn=True)
2148 if not (ran and ran.ok) and docs(ctx, reinstall=True):
2149 return docs(ctx)
2151 return False
2154# noinspection PyUnusedLocal
2155@task()
2156def zen(_: Context) -> None:
2157 """Prints the Zen of Python"""
2158 # noinspection PyUnresolvedReferences
2159 import this # noqa
2162@task()
2163def whoami(ctx: Context) -> None:
2164 """
2165 Debug method to determine user and host name.
2166 """
2167 ran = ctx.run("whoami", hide=True)
2168 i_am = ran.stdout.strip() if ran else ""
2170 ran = ctx.run("hostname", hide=True)
2171 my_location = ran.stdout.strip() if ran else ""
2173 print(f"{i_am} @ {my_location}")
2176@task()
2177def completions(_: Context) -> None:
2178 """
2179 Prints the script to enable shell completions.
2180 """
2181 print("Put this in your .bashrc:")
2182 print("---")
2183 print('eval "$(edwh --print-completion-script bash)"')
2184 print("---")
2187def warn_plugin_version_check(active: str, required: str, name: str) -> bool:
2188 """
2189 Warn if a version is too low (using semver-aware sorting).
2190 """
2191 if parse_version(active) < parse_version(required):
2192 cprint(
2193 f"Note: your `{name}` tool might be outdated ({active} < {required}). "
2194 "To prevent weird behavior, try running `edwh self-update`",
2195 color="yellow",
2196 file=sys.stderr,
2197 )
2198 return False
2199 return True
2202@task(
2203 hookable=True,
2204)
2205def version(ctx: Context) -> None:
2206 """
2207 Show edwh app version and docker + compose version.
2208 """
2209 from ewok.__about__ import __version__ as ewok_version
2211 print("edwh version", edwh_version)
2212 print("ewok version", ewok_version)
2213 print("Python version", sys.version.split(" ")[0])
2214 ctx.run("docker --version")
2215 ctx.run(f"{DOCKER_COMPOSE} version")
2218# for meta tasks such as `plugins` and `self-update`, see meta.py
2221@task(
2222 name="help",
2223 help={
2224 "about": "Plugin/Namespace or Subcommand you would like to see help about. "
2225 "Use an empty string ('') to see help about everything."
2226 },
2227)
2228def show_help(ctx: Context, about: str) -> None:
2229 """
2230 Show helpful information about a plugin or command.
2232 Similar to `edwh {about} --help` but that does not work for whole plugins/namespaces.
2233 """
2234 # first check if 'about' is a plugin/namespace:
2235 if ns := ewok.find_namespace(ctx, about):
2236 info = ns.serialized()
2238 print("--- namespace", ns.name, "---")
2239 print(info["help"] or "")
2241 plugin_commands = []
2242 for subtask in info["tasks"]:
2243 if aliases := subtask["aliases"]:
2244 aliases = ", ".join(aliases)
2245 aliases = f"({aliases})"
2246 else:
2247 aliases = ""
2249 cmd = f"{about}.{subtask['name']}"
2251 plugin_commands.append(" ".join([cmd, aliases, "\t", subtask["help"] or ""]))
2253 print_aligned(plugin_commands)
2254 else:
2255 # just run edwh --help <subcommand>:
2256 ctx.run(f"edwh --help {about}")
2259@task(
2260 name="discover",
2261 help={
2262 "du": "Show disk usage per folder",
2263 "exposes": "Show exposed ports",
2264 "ports": "Show ports",
2265 "host_labels": "Show host clauses from traefik labels",
2266 "short": "Oneline summary",
2267 "show_settings": "show settings per folder",
2268 "as_json": "output json",
2269 },
2270 flags={"show_settings": ("settings", "show-settings"), "as_json": ("j", "json", "as-json")}, # -s is for short
2271)
2272def task_discover(
2273 ctx: Context,
2274 du: bool = False,
2275 exposes: bool = False,
2276 ports: bool = False,
2277 host_labels: bool = True,
2278 short: bool = False,
2279 show_settings: bool = False,
2280 as_json: bool = False,
2281) -> None:
2282 """Discover docker environments per host.
2284 Use ansi2txt to save readable output to a file.
2285 """
2286 return discover(
2287 ctx,
2288 du=du,
2289 exposes=exposes,
2290 ports=ports,
2291 host_labels=host_labels,
2292 short=short,
2293 as_json=as_json,
2294 settings=show_settings,
2295 )
2298@task()
2299def ew_self_update(ctx: Context) -> None:
2300 """Update edwh to the latest version."""
2301 ctx.run("~/.local/bin/edwh self-update")
2302 ctx.run("~/.local/bin/edwh self-update")
2305@task()
2306def migrate(ctx: Context, force: bool = False) -> None:
2307 if force:
2308 clean_flags(ctx)
2310 up(ctx, service=["migrate"], tail=True)
2313@task()
2314def migrations(ctx: Context) -> None:
2315 ctx.run(f"{DOCKER_COMPOSE} run --rm migrate migrate --list")
2318def stop_remove_container(ctx: Context, container_name: str) -> bool:
2319 ran = ctx.run(f"{DOCKER_COMPOSE} rm -vf --stop {container_name}", warn=True)
2320 return bool(ran and ran.ok)
2323def stop_remove_containers(ctx: Context, *container_names: str) -> list[bool]:
2324 return [stop_remove_container(ctx, _) for _ in container_names]
2327@task()
2328def clean_redis(_: Context, db_count: int = 3) -> None:
2329 import redis as r
2331 env = read_dotenv(Path(".env"))
2332 for db in range(db_count):
2333 redis_client = r.Redis("localhost", int(env["REDIS_PORT"]), db)
2334 print(f"Removing {len(redis_client.keys())} keys")
2335 for key in redis_client: # type: ignore
2336 del redis_client[key]
2337 redis_client.close()
2340@task()
2341def clean_flags(_: Context, flag_dir: str = "migrate/flags"):
2342 flag_dir_path = pathlib.Path(flag_dir)
2344 for flag_file in flag_dir_path.glob("*.complete"):
2345 print("removing", flag_file)
2346 flag_file.unlink()
2349@task()
2350def clean_postgres(ctx: Context, yes: bool = False) -> None:
2351 # assumes pgpool with pg-0, pg-1 and optionally pg-stats right now!
2352 yes or confirm(
2353 "Are you sure you want to wipe the database? This can not be undone [yes,NO]",
2354 allowed={"yes"}, # strict yes, not just y !!!
2355 strict=True, # raises RuntimeError
2356 )
2358 # clear backend flag files
2359 clean_flags(ctx)
2361 config = TomlConfig.load()
2362 assert config, "Couldn't set up toml config -> can't continue clean!"
2364 # find the images based on the instances
2365 containers = find_containers_ids(ctx, *config.services_db)
2366 pg_data_volumes = []
2367 for container_name, container_ids in containers.items():
2368 if not container_ids:
2369 # probably missing (such as pg-1, pg-stats in some environments)
2370 continue
2372 for container_id in container_ids:
2373 docker_info = docker_inspect(ctx, container_id)
2374 if not isinstance(docker_info, list):
2375 continue
2377 info = docker_info[0]
2378 pg_data_volumes.extend([mount["Name"] for mount in info["Mounts"] if "Name" in mount])
2380 # stop, remove the postgres instances and remove anonymous volumes
2381 stop_remove_containers(ctx, *containers)
2383 # remove images after containers have been stopped and removed
2384 if pg_data_volumes:
2385 print("removing", pg_data_volumes)
2386 ctx.run("docker volume rm " + " ".join(pg_data_volumes), warn=True)
2387 else:
2388 cprint("No data volumes to remove!", color="yellow")
2391@task(
2392 flags={"clean_all": ("all", "a")},
2393 hookable=True,
2394)
2395def clean(
2396 ctx: Context,
2397 clean_all: bool = False,
2398 db: bool = False,
2399 postgres: bool = False,
2400 redis: bool = False,
2401 yes: bool = False,
2402) -> None:
2403 """Rebuild the databases, possibly rebuild microservices.
2405 Execution:
2406 0. build microservices (all, microservices)
2407 if force_rebuild:
2408 does not use docker-image cache, thus refreshing even with the same backend version.
2409 use this is you wish to rebuild the same backend. Easier and faster to use fix: or perf: in the backend...
2410 1. stopping postgres instances (all, db, postgres)
2411 2. removing volumes (all, db, postgres)
2412 3. rebooting postgres instances (all, db, postgres)
2413 4. ~~purge redis instances (all, redis)~~ IGNORED
2415 Removes all ../backend_config/*.complete flags to allow migrate to function properly
2416 """
2417 print("-------------------CLEAN -------------------------")
2418 if clean_all or db or postgres:
2419 clean_postgres(ctx, yes=yes)
2421 if clean_all or redis:
2422 clean_redis(ctx)
2425@task(aliases=("whipe-db",), flags={"clean_all": ("all", "a")})
2426def wipe_db(ctx: Context, clean_all: bool = False, flag_path: str = "migrate/flags", yes: bool = False) -> None:
2427 """
2428 Wipes postgres volumes.
2429 Does not start migrate automatically, use `edwh wipe-db migrate up` for a full recovery flow.
2431 When using a whitelabel-based environment,
2432 you may also use `edwh local.recover-devdb` (from ./migrate/data/snapshot)
2433 """
2434 # 1 + 2. just 'create' without starting anything:
2435 ctx.run(f"{DOCKER_COMPOSE} create")
2437 # 3. start cleaning up
2438 for p in Path(flag_path).glob("migrate-*.complete"):
2439 p.unlink()
2441 clean(ctx, db=True, clean_all=clean_all, yes=yes)
2442 down(ctx) # remove old containers too
2445@task()
2446def show_config(_: Context) -> None:
2447 """
2448 Show the current values from .toml after loading.
2449 """
2450 config = TomlConfig.load()
2451 cprint(f"TomlConfig: {json.dumps(config.__dict__, default=str, indent=2) if config else 'None'}")
2454@task()
2455def change_config(c: Context) -> None:
2456 """
2457 Change the settings in .toml
2458 """
2459 build_toml(c, overwrite=True)
2462@task()
2463def debug(_: Context) -> None:
2464 print(get_env_value("IS_DEBUG", "0"))
2467@task(aliases=("ew",))
2468def edwh(_: Context) -> None:
2469 """
2470 Do absolutely nothing.
2472 For oopsies like `ew ew up logs`
2473 """
2474 print("Hehe you silly goose", file=sys.stderr)
2477@task()
2478def sleep(_: Context, n: str) -> None:
2479 try:
2480 totaltime = int(n)
2481 except ValueError as e:
2482 raise TypeError("`ew sleep <n: int>` requires an amount of seconds to sleep.") from e
2484 for remaining in range(totaltime, 0, -1):
2485 print("\r", f"Sleeping for: {remaining} seconds", end=" ", flush=True)
2486 time.sleep(1)
2488 print("\r", "Sleeping for: 0 seconds", end="\n")
2491def find_ruff() -> str:
2492 """
2493 Use ruff's own logic to find the required binary.
2494 """
2495 from ruff import __main__ as ruff
2497 return ruff.find_ruff_bin()
2500def find_ty() -> str:
2501 """Use Ty's own logic to find the required binary."""
2502 from ty import find_ty_bin
2504 return find_ty_bin()
2507def enabled_lint_tools() -> dict[str, bool]:
2508 """Return the enabled lint tools for the current project.
2510 Projects can opt out of either tool independently in ``pyproject.toml``:
2512 [tool.edwh.lint]
2513 ruff = false
2514 ty = false
2515 """
2516 pyproject = Path("pyproject.toml")
2517 if not pyproject.exists():
2518 return {"ruff": True, "ty": True}
2520 config = tomllib.loads(pyproject.read_text())
2521 lint_config = config.get("tool", {}).get("edwh", {}).get("lint", {})
2523 return {tool: lint_config.get(tool, True) is not False for tool in ("ruff", "ty")}
2526type OutputMode = t.Literal["ci", "cli", "json"]
2529class LintOutput(t.TypedDict):
2530 mode: OutputMode
2531 results: dict[str, bool]
2534def _render_lint_result(_: Context, result: LintOutput) -> str | None:
2535 if not isinstance(result, dict):
2536 return None
2538 if result.get("mode", "cli") != "json":
2539 return None
2541 data = dict(result.get("results", {}))
2543 print(json.dumps(data, indent=2, sort_keys=True))
2545 if not all(data.values()):
2546 raise SystemExit(1)
2548 return None
2551def _print_lint_status(tool: str, ok: bool, output: OutputMode) -> None:
2552 if output == "ci":
2553 print(f"{'✅' if ok else '❌'} {tool}")
2554 else:
2555 color: Color = "green" if ok else "red"
2556 cprint(f"⬤ {tool}", color=color)
2559def run_command_with_output(
2560 ctx: Context,
2561 command: str,
2562 mode: OutputMode = "cli",
2563 tool: str | None = None,
2564) -> bool:
2565 """Run a command, capture its output, and replay it to the current terminal."""
2567 result = ctx.run(command, hide=True, warn=True)
2568 ok = bool(result and result.ok)
2569 should_print = not ok and mode != "json"
2571 if should_print and result is not None and result.stdout:
2572 print(result.stdout, end="")
2574 if should_print and result is not None and result.stderr:
2575 print(result.stderr, end="", file=sys.stderr)
2577 if tool and mode in {"cli", "ci"}:
2578 _print_lint_status(tool, ok, mode)
2580 return ok
2583@task(
2584 hookable=True,
2585 help={"output": "output mode: cli, ci, or json"},
2586 result_renderer=_render_lint_result,
2587)
2588def lint(
2589 ctx: Context,
2590 directory: Optional[str] = None,
2591 select: str = "",
2592 fix: bool = False,
2593 output: OutputMode = "cli",
2594) -> LintOutput:
2595 """
2596 Lint code with `ruff` and `ty`.
2598 Disable either tool for a project with ``[tool.edwh.lint]`` in
2599 ``pyproject.toml``. Both are enabled by default.
2601 Args:
2602 ctx: invoke context
2603 directory: where to look for code
2604 select: specific lints to check
2605 fix: try to fix (some) issues automatically
2606 output: render output as cli, ci, or json
2607 """
2608 directory = directory or "."
2609 output = t.cast(OutputMode, output.lower())
2611 if output not in {"cli", "ci", "json"}:
2612 raise ValueError(f"Invalid --output value: {output}. Expected one of: cli, ci, json.")
2614 enabled_tools = enabled_lint_tools()
2615 results: dict[str, bool] = {}
2617 if enabled_tools["ruff"]:
2618 ruff = find_ruff()
2619 command = [ruff, "check", directory, "--quiet"]
2620 if select:
2621 command.extend(("--select", select))
2622 if fix:
2623 command.append("--fix")
2625 results["linting (ruff)"] = run_command_with_output(ctx, shlex.join(command), mode=output, tool="ruff")
2627 if enabled_tools["ty"]:
2628 ty = find_ty()
2629 results["type checking (ty)"] = run_command_with_output(
2630 ctx,
2631 shlex.join([ty, "check", directory]),
2632 mode=output,
2633 tool="ty",
2634 )
2636 return {"mode": output, "results": results}
2639@task(aliases=("format",), hookable=True)
2640def fmt(
2641 ctx: Context,
2642 isort: bool = True,
2643 ioptimize: bool = False,
2644 reformat: bool = True,
2645 directory: Optional[str] = None,
2646 file: Optional[str] = None,
2647 quiet: bool = False,
2648):
2649 """
2650 Format your Python code with `ruff`, including import sorting (isort).
2652 `ioptimize` would remove unused imports, but that functionality in `ruff` doesn't seem to work right now
2653 -> so only display problems for now.
2655 `file` and `directory` have the same behavior, the different names are there for sugar.
2656 """
2657 if file and directory:
2658 raise ValueError("Conflicting arguments --file and --directory. Please pick one, the behavior is the same.")
2660 target = directory or file or "."
2662 ruff = find_ruff()
2664 color: Color
2666 if isort:
2667 color = "green" if run_pty_ok(ctx, ruff, f"check --select I --fix {target} --quiet") else "red"
2668 cprint("⬤ isort", color=color)
2670 if reformat:
2671 # note: ruff format --quiet also hides what's wrong, so instead pipe stdout to dev null and only show stderr:
2672 color = "green" if run_pty_ok(ctx, ruff, f"format {target} > /dev/null") else "red"
2673 cprint("⬤ reformat", color=color)
2675 if not quiet and not ioptimize:
2676 # print out unused imports:
2677 try:
2678 # grep remove the --fix suggestion since we have other cli args;
2679 # grep piping removes the nice coloring unless we force it;
2680 # pipefall would forward ruff's exit code but if grep has no output, it exits with 1.
2681 # so we do this fuckery:
2682 ctx.run(
2683 f"""
2684 ruff_output=$(FORCE_COLOR=1 {ruff} check --select F401 {target})
2685 ruff_exit=$?
2686 echo "$ruff_output" | grep -v -E '(`--fix`|^All checks passed!$)' || true
2687 exit $ruff_exit
2688 """,
2689 pty=True,
2690 )
2691 except invoke.exceptions.UnexpectedExit:
2692 cprint(
2693 "Hint: unused imports can be removed with --ioptimize; this check can be skipped with --quiet",
2694 "blue",
2695 )
2697 elif ioptimize:
2698 # else, autofix F401 = unused-import
2699 color = "green" if run_pty_ok(ctx, ruff, f"check --select F401 {target} --fix --quiet") else "red"
2700 cprint("⬤ ioptimize", color=color)