jmri_web.catalog_autoload

  1"""Auto-load the decoder catalog at startup when ``JMRI_DECODER_XML_ROOT``
  2is set.
  3
  4The manual load path (``POST /programmer/catalog/load``) runs a
  5worker thread + progress polling because a fresh scan of JMRI's
  6decoder tree takes 30–60 s on a Pi. We reuse that pattern here:
  7the ``schedule_autoload`` helper kicks off the same worker in the
  8background and returns immediately, so uvicorn's lifespan startup
  9doesn't block on it.
 10
 11Progress is recorded on ``handle.ctx._catalog_load_progress`` in the
 12same shape the manual load uses (``{loaded, total, done, error,
 13directory}``). The programmer page renders the progress bar when
 14this dict is present and ``done`` is False — so a user who lands on
 15``/programmer`` while the autoload is running sees "Loading catalog…"
 16with a live thermometer instead of the stale "Catalog empty" hint
 17that would tempt them to click Load again.
 18"""
 19from __future__ import annotations
 20
 21import asyncio
 22import logging
 23import pathlib
 24from typing import Any
 25
 26_log = logging.getLogger(__name__)
 27
 28
 29def _load_catalog_sync(
 30    target: pathlib.Path,
 31    progress: dict[str, Any] | None = None,
 32) -> Any:
 33    """Do the actual catalog scan. Blocking — call from a worker
 34    thread. If ``progress`` is passed, it's updated in place with
 35    per-file counters so the progress-poll endpoint can render the
 36    thermometer live."""
 37    from jmri_core.persistence.decoder_xml import (  # noqa: PLC0415
 38        iter_decoder_index,
 39    )
 40    from jmri_decoderpro import DecoderCatalog  # noqa: PLC0415
 41
 42    catalog = DecoderCatalog()
 43    seen_files: set[pathlib.Path] = set()
 44    for entry in iter_decoder_index(target):
 45        catalog.add_index_entry(entry)
 46        if entry.source_file not in seen_files:
 47            seen_files.add(entry.source_file)
 48            if progress is not None:
 49                progress["loaded"] = len(seen_files)
 50    if progress is not None and progress.get("total"):
 51        # Match the total so the thermometer reads 100% once we're
 52        # past the scan phase (before the follow-up mfg_ids load).
 53        progress["loaded"] = progress["total"]
 54
 55    programmers_dir = target.parent / "programmers"
 56    if programmers_dir.is_dir():
 57        from jmri_core.persistence.programmer_xml import (  # noqa: PLC0415
 58            iter_programmer_configs,
 59        )
 60        catalog.programmers = {
 61            pc.name: pc for pc in iter_programmer_configs(programmers_dir)
 62        }
 63
 64    index_path = target.parent / "decoderIndex.xml"
 65    if index_path.is_file():
 66        from jmri_decoderpro.catalog import _load_mfg_ids  # noqa: PLC0415
 67        catalog.mfg_ids = _load_mfg_ids(index_path)
 68    return catalog
 69
 70
 71def schedule_autoload(handle: Any) -> asyncio.Task[None] | None:
 72    """If ``JMRI_DECODER_XML_ROOT`` is set to a real directory, kick
 73    off a background catalog load. Returns the task so callers can
 74    await it in tests, or ``None`` when no autoload was scheduled
 75    (env unset, dir missing, decoderpro not installed).
 76
 77    Idempotent — a subsequent manual load through the /programmer
 78    tab replaces the catalog atomically. Errors are logged, marked
 79    on the progress dict, and swallowed so a bad env var doesn't
 80    break server startup.
 81    """
 82    from jmri_core.env import decoder_xml_root  # noqa: PLC0415
 83
 84    target = decoder_xml_root()
 85    if target is None:
 86        return None
 87    if not target.is_dir():
 88        _log.warning(
 89            "JMRI_DECODER_XML_ROOT=%s: not a directory; skipping autoload",
 90            target,
 91        )
 92        return None
 93
 94    # Precount XML files so the thermometer has a denominator. This
 95    # is a synchronous glob — cheap on a Pi even for the full
 96    # ~1200-file JMRI decoder tree (single dirent walk, no parsing).
 97    try:
 98        total = sum(1 for _ in target.rglob("*.xml"))
 99    except OSError as e:
100        _log.warning("autoload: couldn't precount %s: %s", target, e)
101        total = 0
102
103    progress: dict[str, Any] = {
104        "loaded": 0,
105        "total": total,
106        "done": False,
107        "error": None,
108        "directory": str(target),
109    }
110    handle.ctx._catalog_load_progress = progress  # type: ignore[attr-defined]
111
112    async def _run() -> None:
113        try:
114            catalog = await asyncio.to_thread(
115                _load_catalog_sync, target, progress,
116            )
117        except Exception as e:  # noqa: BLE001
118            _log.exception(
119                "decoder catalog autoload failed for %s", target,
120            )
121            progress["error"] = str(e)
122            progress["done"] = True
123            return
124        handle.ctx._decoder_catalog = catalog  # type: ignore[attr-defined]
125        progress["done"] = True
126        _log.info(
127            "decoder catalog auto-loaded from %s: %d entr%s",
128            target, len(catalog),
129            "y" if len(catalog) == 1 else "ies",
130        )
131
132    return asyncio.create_task(_run(), name="decoder-catalog-autoload")