"""Reusable discovery registry for values stored in Python modules."""
from __future__ import annotations
import importlib
import pkgutil
import os
import warnings
from threading import RLock
from types import MappingProxyType
[docs]
class RegistryEntryNotFound(KeyError):
"""A requested registry key has no discoverable module."""
[docs]
class RegistryDefinitionError(RuntimeError):
"""A discovered registry module or registry configuration is malformed."""
[docs]
class RegistryLoadWarning(UserWarning):
"""A definition was skipped while enumerating registry entries."""
[docs]
class ModuleRegistry:
"""Lazily discover and load one named value from package modules.
Module names are discovered with :mod:`pkgutil` without importing the
modules. Values are imported and materialized only when requested.
Subclasses can customize extraction, validation and aliasing without
duplicating discovery machinery.
Discovery and cache materialization are synchronized so concurrent first
lookups observe the same cached objects. Module imports themselves remain
outside the registry lock and rely on Python's import synchronization.
``attribute`` names the value exported by each registry module. Exported
callables are returned unchanged unless ``factory=True`` is requested
explicitly.
"""
[docs]
def __init__(
self,
package,
attribute=None,
*,
aliases=None,
include=None,
factory=False,
):
self.package = package
self.attribute = attribute
self._include = include or (lambda name: True)
self._factory = factory
self.aliases = MappingProxyType(self._normalize_aliases(aliases or {}))
self._lock = RLock()
self._module_names = None
self._modules = {}
self._values = {}
self._warned_load_failures = set()
[docs]
@staticmethod
def normalize_name(name):
return str(name).upper()
[docs]
def _normalize_aliases(self, aliases):
normalized = {}
for alias, target in aliases.items():
alias = self.normalize_name(alias)
target = self.normalize_name(target)
if alias in normalized:
raise RegistryDefinitionError(
f"Duplicate normalized registry alias {alias!r}"
)
normalized[alias] = target
return normalized
[docs]
def _validate_aliases(self, module_names):
for alias, target in self.aliases.items():
if alias in module_names:
raise RegistryDefinitionError(
f"Registry alias {alias!r} shadows a module in "
f"{self.package.__name__}"
)
if target not in module_names:
raise RegistryDefinitionError(
f"Registry alias {alias!r} targets {target!r}, which is not "
f"a canonical module in {self.package.__name__}"
)
[docs]
def _discover_module_names(self):
with self._lock:
if self._module_names is None:
discovered = {}
for item in sorted(
pkgutil.iter_modules(self.package.__path__),
key=lambda item: item.name,
):
if item.name.startswith("_") or not self._include(item.name):
continue
name = self.normalize_name(item.name)
if name in discovered:
raise RegistryDefinitionError(
f"Duplicate module registry key {name!r} in "
f"{self.package.__name__}"
)
discovered[name] = item.name
self._validate_aliases(discovered)
self._module_names = MappingProxyType(discovered)
return self._module_names
[docs]
def _load_module(self, name):
with self._lock:
if name in self._modules:
return self._modules[name]
module_name = self._discover_module_names().get(name)
if module_name is None:
raise RegistryEntryNotFound(name)
module = importlib.import_module(f".{module_name}", self.package.__name__)
with self._lock:
return self._modules.setdefault(name, module)
[docs]
def value_from_module(self, name, module):
"""Extract the registered value from a loaded module."""
if self.attribute is None:
raise NotImplementedError
try:
value = vars(module)[self.attribute]
except KeyError as exc:
raise RegistryDefinitionError(
f"Registry module {module.__name__!r} in {module.__file__!r} "
f"does not define required export {self.attribute!r}"
) from exc
if self._factory and callable(value):
value = value()
return value
[docs]
def is_user_defined(self, module):
return os.path.dirname(module.__file__) != self.package.__path__[0]
[docs]
def prepare_value(self, name, module, value):
"""Validate or transform a freshly loaded value."""
return value
[docs]
def alias_value(self, alias, value):
"""Return the value exposed under *alias*."""
return value
[docs]
def _load_value(self, name):
with self._lock:
if name in self._values:
return self._values[name]
module = self._load_module(name)
with self._lock:
if name in self._values:
return self._values[name]
value = self.value_from_module(name, module)
value = self.prepare_value(name, module, value)
self._values[name] = value
return value
def __getitem__(self, name):
requested = self.normalize_name(name)
with self._lock:
if requested in self._values:
return self._values[requested]
module_names = self._discover_module_names()
canonical = self.aliases.get(requested, requested)
if canonical not in module_names:
raise RegistryEntryNotFound(requested)
value = self._load_value(canonical)
if requested == canonical:
return value
with self._lock:
if requested in self._values:
return self._values[requested]
value = self.alias_value(requested, value)
self._values[requested] = value
return value
def __contains__(self, name):
"""Return whether *name* is discoverable without importing its module."""
requested = self.normalize_name(name)
module_names = self._discover_module_names()
return requested in module_names or requested in self.aliases
[docs]
def names(self, *, include_aliases=True, name_contains=None):
"""Return discovered names without importing registry modules."""
module_names = self._discover_module_names()
if name_contains is not None:
module_names = (i for i in module_names if name_contains in i)
names = list(module_names)
if include_aliases:
names.extend(self.aliases)
return tuple(names)
[docs]
def values(self, *, include_aliases=False, name_contains=None):
"""Load and return registered values."""
return tuple(
self[name]
for name in self.names(include_aliases=include_aliases, name_contains=name_contains)
)
[docs]
def iter_valid_values(self, *, names=None, include_aliases=False,
name_contains=None, condition=None):
"""Load entries in an enumeration, warning and skipping invalid ones.
A direct lookup and :meth:`values` remain strict. The optional names
restrict which entries are materialized, preserving lazy lookup for
filtered enumerations. Each failing entry is warned about at most once.
"""
if names is None:
names = self.names(include_aliases=include_aliases, name_contains=name_contains)
for name in names:
try:
value = self[name]
except Exception as exc:
canonical = self.aliases.get(self.normalize_name(name), self.normalize_name(name))
with self._lock:
warn = canonical not in self._warned_load_failures
self._warned_load_failures.add(canonical)
if warn:
warnings.warn(
f"Skipping invalid registry entry {self.package.__name__}.{canonical.lower()}: "
f"{type(exc).__name__}: {exc}",
RegistryLoadWarning,
stacklevel=2,
)
continue
if not condition or condition(value):
yield value
@property
def modules(self):
"""Import canonical registry modules without materializing values."""
for name in self.names(include_aliases=False):
self._load_module(name)
with self._lock:
return dict(self._modules)