jmri_core.persistence.decoder_xml

   1"""Decoder definition XML loader — read-only.
   2
   3Each file under ``xml/decoders/*.xml`` describes one decoder family (one or
   4more models) plus the CVs each model exposes. JMRI ships 881 such files.
   5
   6Stage 2 ships a **read-only** parser: enough to enumerate manufacturers,
   7families, models, and CV variables, so Stage 3+ can serve roster data and
   8Stage 8 (DecoderPro) can build a UI on top. Editing/writing decoder XML is
   9not part of this stage — JMRI itself is the source of truth for these files.
  10
  11The model intentionally captures only the major attributes. The full schema
  12``decoder-4-15-2.xsd`` declares dozens of element types; rather than model
  13every one, we expose:
  14
  15* family + model identity (vendor, family name, model name, connector type);
  16* per-model attributes JMRI cares about for roster matching;
  17* a flat list of :class:`DecoderVariable` entries (CV definitions);
  18* the raw lxml ``Element`` via :attr:`DecoderFile.tree` for callers who need
  19  to reach into uncaptured corners (programmer pane layouts, etc.).
  20"""
  21from __future__ import annotations
  22
  23import logging
  24import pathlib
  25from collections.abc import Iterable, Iterator
  26from dataclasses import dataclass
  27from enum import StrEnum
  28
  29from lxml import etree
  30from pydantic import BaseModel, ConfigDict, Field
  31
  32_log = logging.getLogger(__name__)
  33
  34
  35class DecoderOutput(BaseModel):
  36    """Function-output mapping declared by a model."""
  37    name: str
  38    label: str = ""
  39
  40
  41# ----- variable value-type modelling ------------------------------------
  42
  43
  44class VariableKind(StrEnum):
  45    """The value-type child element declared inside a ``<variable>``.
  46
  47    ``UNKNOWN`` is the parser's fallback when a variable declares something
  48    we don't model yet; downstream code treats it as an opaque byte.
  49    """
  50    DECIMAL         = "decimal"          # <decVal>
  51    HEXADECIMAL     = "hexadecimal"      # <hexVal>
  52    ENUM            = "enum"             # <enumVal>/<enumChoice>
  53    SPEED_TABLE     = "speed_table"      # <speedTable>
  54    SHORT_ADDRESS   = "short_address"    # <shortAddressVal> (CV1)
  55    LONG_ADDRESS    = "long_address"     # <longAddressVal>  (CV17+CV18)
  56    SPLIT           = "split"            # <splitVal>
  57    SPLIT_HEX       = "split_hex"        # <splitHexVal>
  58    SPLIT_DATETIME  = "split_datetime"   # <splitDateTimeVal>
  59    COMPOSITE       = "composite"        # <compositeVal>
  60    CONSTANT        = "constant"         # <constant>
  61    UNKNOWN         = "unknown"
  62
  63
  64class EnumChoice(BaseModel):
  65    """One ``<enumChoice value="V">text</enumChoice>`` option.
  66
  67    JMRI allows the ``value`` attribute to be omitted, in which case the
  68    positional index (0, 1, 2, …) is the wire value. The parser fills
  69    that in so downstream code can treat all choices uniformly.
  70    """
  71    value: int
  72    choice: str = ""
  73    choice_by_lang: dict[str, str] = Field(default_factory=dict)
  74
  75
  76class CompositeSetting(BaseModel):
  77    """One ``<compositeSetting label="X" value="Y"/>`` inside a compositeChoice.
  78
  79    ``label`` references another variable via its ``item`` name; picking
  80    this composite choice writes ``value`` into that referenced variable.
  81    """
  82    label: str
  83    value: int
  84
  85
  86class CompositeChoice(BaseModel):
  87    """One ``<compositeChoice choice="X"><compositeSetting/>…</compositeChoice>``.
  88
  89    Composite variables are meta-variables — a user pick fans out into
  90    multiple sub-variable writes. The write path needs the session-level
  91    queue from step 4 to batch those atomically; step 2 just models the
  92    structure.
  93    """
  94    choice: str
  95    choice_by_lang: dict[str, str] = Field(default_factory=dict)
  96    settings: list[CompositeSetting] = Field(default_factory=list)
  97
  98
  99class CvBitMask(BaseModel):
 100    """Parsed ``<variable mask="XXVXXXXX">`` notation.
 101
 102    JMRI masks are written MSB → LSB with ``V`` marking bits this
 103    variable owns and ``X`` marking don't-cares. ``"XXVXXXXX"`` means
 104    bit 5 (``0b00100000``).
 105
 106    * ``shift`` — position of the LSB of the ``V``-region (from bit 0)
 107    * ``width`` — number of ``V`` bits
 108    * ``mask_value`` — the raw byte mask for the ``V``-region
 109    """
 110    model_config = ConfigDict(frozen=True)
 111
 112    raw: str = ""
 113    shift: int = 0
 114    width: int = 8
 115
 116    @classmethod
 117    def parse(cls, s: str | None) -> CvBitMask:
 118        """Turn a JMRI mask string into a structured mask.
 119
 120        Empty / all-``X`` / missing masks give a full-byte mask (shift=0,
 121        width=8) — that's what JMRI treats as "no mask" and it means the
 122        variable owns the whole byte.
 123        """
 124        if not s:
 125            return cls(raw="", shift=0, width=8)
 126        s = s.strip()
 127        if not s or "V" not in s:
 128            return cls(raw=s, shift=0, width=len(s) or 8)
 129        total = len(s)
 130        first_v = s.find("V")
 131        last_v = s.rfind("V")
 132        shift = total - 1 - last_v
 133        width = last_v - first_v + 1
 134        return cls(raw=s, shift=shift, width=width)
 135
 136    @property
 137    def mask_value(self) -> int:
 138        """The byte mask that isolates this variable's V-region."""
 139        return ((1 << self.width) - 1) << self.shift
 140
 141    def extract(self, byte: int) -> int:
 142        """Read this variable's value out of a raw byte."""
 143        return (byte & self.mask_value) >> self.shift
 144
 145    def apply(self, byte: int, value: int) -> int:
 146        """Stuff *value* into the V-region of *byte*, preserving other bits.
 147
 148        The read-modify-write step the caller performs before writing
 149        back: fetch the current byte, call ``apply(byte, new_value)``,
 150        write the resulting byte. Values wider than ``width`` bits are
 151        truncated to the region size.
 152        """
 153        room = (1 << self.width) - 1
 154        return (byte & ~self.mask_value) | ((value & room) << self.shift)
 155
 156
 157class SplitCv(BaseModel):
 158    """Split-CV addressing parsed from ``<variable CV="…">`` attribute strings.
 159
 160    Three JMRI-supported shapes:
 161
 162    * ``"1"`` — one CV, single byte
 163    * ``"17.18"`` — long-address pattern: CV17 is the high byte
 164      (with ``0xC0`` bits set), CV18 is the low byte
 165    * ``"0.255.261"`` — bank/index/base indexed access (ESU-style)
 166    * ``"0.255.261:4"`` — same as above, 4 consecutive bytes
 167
 168    ``cv_list`` gives the ordered CVs the variable touches, in
 169    high-byte-first order (the read/write path iterates them).
 170    """
 171    model_config = ConfigDict(frozen=True)
 172
 173    cv_attr: str
 174    primary_cv: int = 0
 175    byte_count: int = 1
 176    bank: int | None = None
 177    index: int | None = None
 178
 179    @classmethod
 180    def parse(cls, s: str) -> SplitCv:
 181        if not s:
 182            return cls(cv_attr="", primary_cv=0, byte_count=1)
 183        s = s.strip()
 184        if ":" in s:
 185            head, count = s.split(":", 1)
 186            try:
 187                byte_count = max(1, int(count))
 188            except ValueError:
 189                byte_count = 1
 190        else:
 191            head = s
 192            byte_count = 1
 193        parts = head.split(".")
 194        try:
 195            nums = [int(p) for p in parts]
 196        except ValueError:
 197            return cls(cv_attr=s, primary_cv=0, byte_count=byte_count)
 198        if len(nums) == 1:
 199            return cls(cv_attr=s, primary_cv=nums[0], byte_count=byte_count)
 200        if len(nums) == 2:
 201            # Two adjacent CVs — the long-address pattern (CV17.CV18)
 202            # or any two-byte pair.
 203            return cls(
 204                cv_attr=s, primary_cv=nums[0],
 205                byte_count=max(byte_count, 2),
 206            )
 207        # 3+ parts: bank.index.base
 208        return cls(
 209            cv_attr=s,
 210            bank=nums[0], index=nums[1],
 211            primary_cv=nums[2],
 212            byte_count=byte_count,
 213        )
 214
 215    @property
 216    def cv_list(self) -> list[int]:
 217        """The CVs this variable touches, in high-byte-first order."""
 218        return [self.primary_cv + i for i in range(self.byte_count)]
 219
 220
 221class DecoderVariable(BaseModel):
 222    """One CV (or CV-group) variable as declared in ``<variables>``.
 223
 224    Backwards-compatible fields (``cv`` through ``ops_only``) stay
 225    exactly as parsed in earlier stages. Step 2 adds:
 226
 227    * ``kind`` — which value-type child element the variable declared
 228    * ``bit_mask`` — parsed :class:`CvBitMask` (never None; a variable
 229      without a mask still has a full-byte mask)
 230    * ``enum_choices`` — ordered choices for ENUM variables
 231    * ``composite_choices`` — ordered choices for COMPOSITE variables
 232    * ``split`` — parsed :class:`SplitCv` for SPLIT/SPLIT_HEX/etc
 233    * ``constant_value`` — for the CONSTANT kind
 234    * ``read_only`` / ``write_only`` — attribute flags
 235    * ``tooltip`` — hover text
 236    """
 237    model_config = ConfigDict(populate_by_name=True)
 238
 239    cv: str = Field(alias="CV", default="")
 240    item: str = ""
 241    # ``<variable default="N">`` — the *human* default value the
 242    # decoder ships with (for a masked-bit variable, the masked
 243    # value; for a decimal variable, the whole byte). Parsed to an
 244    # ``int`` at load time via :func:`_parse_default_int` so downstream
 245    # consumers (default-overlay in the pane render, diff-filter in
 246    # LocoCvStore, CV7 identity match) don't have to re-parse on
 247    # every access. ``None`` when the attribute is absent or the value
 248    # isn't a plain integer.
 249    default: int | None = None
 250    mask: str | None = None
 251    min: str | None = None
 252    max: str | None = None
 253    label: str = ""
 254    ops_only: bool = False
 255
 256    # step-2 additions
 257    kind: VariableKind = VariableKind.UNKNOWN
 258    bit_mask: CvBitMask = Field(default_factory=CvBitMask)
 259    enum_choices: list[EnumChoice] = Field(default_factory=list)
 260    composite_choices: list[CompositeChoice] = Field(default_factory=list)
 261    split: SplitCv | None = None
 262    constant_value: int | None = None
 263    read_only: bool = False
 264    write_only: bool = False
 265    tooltip: str = ""
 266    # SPEED_TABLE only: number of speed-step entries the table covers.
 267    # NMRA standard is 28 (CV67..CV94); some decoders declare a 14-step
 268    # or 128-step variant via ``<speedTable entries="N">``.
 269    speed_table_entries: int = 28
 270    # ``<variable include="pid1,pid2,…">`` / ``exclude="…"``: show this
 271    # variable only for models whose ``productID`` is in the include
 272    # list (or NOT in the exclude list). Empty means "no filter".
 273    # SoundTraxx Tsunami2 uses this heavily so a single decoder file
 274    # can serve dozens of model revisions: e.g. HEP Mode / Turbine
 275    # Start-Stop / Stop Request Bell share a CV and take turns being
 276    # applicable per productID.
 277    include_product_ids: list[str] = Field(default_factory=list)
 278    exclude_product_ids: list[str] = Field(default_factory=list)
 279    # ``<qualifier><variableref>X</variableref><relation>Y</relation>
 280    # <value>N</value></qualifier>`` — runtime visibility gate. All
 281    # qualifiers must pass (AND) for the variable to be shown. Common
 282    # relations: ``eq``, ``ne``, ``gt``, ``ge``, ``lt``, ``le``,
 283    # ``exists``. SoundTraxx uses this to split a shared CV across
 284    # multiple named variables depending on another CV's current
 285    # value (HEP Mode / HEP Mode2 / Turbine Start-Stop pattern).
 286    qualifiers: list[dict[str, str]] = Field(default_factory=list)
 287
 288
 289class DecoderModel(BaseModel):
 290    """One model within a family."""
 291    model: str
 292    connector: str = ""
 293    show_empty_panes: str | None = None
 294    max_input_volts: str | None = None
 295    max_motor_current: str | None = None
 296    outputs: list[DecoderOutput] = Field(default_factory=list)
 297    protocols: list[str] = Field(default_factory=list)
 298    size_length: str | None = None
 299    size_width: str | None = None
 300    size_height: str | None = None
 301    size_units: str | None = None
 302    # ``<model productID="…"/>`` — SoundTraxx, Digitrax, etc. use this
 303    # to filter variables per specific decoder revision. When a
 304    # ``<variable>`` carries ``include="pid1,pid2,…"`` or ``exclude="…"``,
 305    # the UI shows/hides it based on whether this model's productID is
 306    # in the list.
 307    product_id: str = ""
 308
 309
 310class DecoderFamily(BaseModel):
 311    """One family containing one or more :class:`DecoderModel` entries."""
 312    name: str
 313    mfg: str = ""
 314    models: list[DecoderModel] = Field(default_factory=list)
 315    # Family-level ``<output>`` declarations. Some decoders (SoundTraxx
 316    # Tsunami2, Digitrax, …) put ``<output name="N" label="…"/>`` after
 317    # all the ``<model>`` elements so the labels apply to every model
 318    # in the family. Callers that render function-map column headers
 319    # should fall back here when a model has no outputs of its own.
 320    outputs: list[DecoderOutput] = Field(default_factory=list)
 321
 322
 323class DecoderPane(BaseModel):
 324    """One JMRI programmer pane declared inside a decoder file.
 325
 326    Panes are the tabs in DecoderPro's programmer UI (Basic, Motor,
 327    Sound, ...). A decoder file adds *extra* panes on top of the
 328    standard set provided by the active programmer definition (which
 329    lives in ``xml/programmers/*.xml`` — a separate loader).
 330
 331    ``items`` is the ordered list of variable ``item`` names referenced
 332    by ``<display item="...">`` anywhere in the pane. Change-tracking
 333    and pane-scoped read/write queues iterate this list.
 334
 335    ``raw_element`` preserves the underlying lxml element so a later
 336    rendering step can reproduce the ``<column>``/``<row>``/``<label>``/
 337    ``<separator>`` structure faithfully.
 338    """
 339    model_config = ConfigDict(arbitrary_types_allowed=True)
 340
 341    name: str = ""
 342    # ``xml:lang`` → localised pane name. The default name (no lang
 343    # attribute) stays in ``name``; other locales go here so a future
 344    # UI can pick the user's language.
 345    name_by_lang: dict[str, str] = Field(default_factory=dict)
 346    # Ordered list of variable ``item`` names referenced by <display>
 347    # elements anywhere inside the pane. Duplicates are preserved
 348    # because JMRI's UI shows one row per <display>, not one row per
 349    # unique item.
 350    items: list[str] = Field(default_factory=list)
 351    # ``<cvtable/>`` marker anywhere in the pane — the CVs tab that
 352    # DecoderPro's "CVs" pane uses to expose raw CV read/write.
 353    has_cvtable: bool = False
 354    # ``<dccaddress/>`` marker — a first-class widget shown by the
 355    # Basic pane at the top.
 356    has_dcc_address: bool = False
 357    # ``<speedtable/>`` widget marker. JMRI's Speed Table pane doesn't
 358    # list per-step ``<display>`` items; instead the whole 28-step
 359    # curve is drawn by this widget. When set, the render layer expands
 360    # the decoder's SPEED_TABLE variable into per-step CV rows.
 361    has_speed_table: bool = False
 362    # ``<fnmapping/>`` marker — the Function Map pane's checkbox
 363    # matrix. Like ``<cvtable/>``, it doesn't reference variables
 364    # via ``<display>``; the renderer collects ``Fx controls output N``
 365    # variables from the decoder's variable list.
 366    has_fnmapping: bool = False
 367    # Raw pane element for step-3 rich rendering.
 368    raw_element: etree._Element | None = None
 369
 370
 371class DecoderFile(BaseModel):
 372    """One ``xml/decoders/*.xml`` file parsed into a model.
 373
 374    The ``raw_xml`` field is the original bytes (for byte-level operations);
 375    individual element access is via the model attributes. ``includes``
 376    captures any ``xi:include`` references for later resolution.
 377    """
 378    model_config = ConfigDict(arbitrary_types_allowed=True)
 379
 380    path: pathlib.Path | None = None
 381    family: DecoderFamily | None = None
 382    variables: list[DecoderVariable] = Field(default_factory=list)
 383    # Decoder-declared programmer panes, in source order. Empty for
 384    # decoder files that don't declare any — the programmer definition
 385    # provides the standard tab set separately.
 386    panes: list[DecoderPane] = Field(default_factory=list)
 387    includes: list[str] = Field(default_factory=list)
 388
 389
 390XINCLUDE_NS = "{http://www.w3.org/2001/XInclude}"
 391XML_LANG_ATTR = "{http://www.w3.org/XML/1998/namespace}lang"
 392
 393
 394def load_decoder_bytes(
 395    data: bytes,
 396    path: pathlib.Path | None = None,
 397    *,
 398    xml_root: pathlib.Path | None = None,
 399) -> DecoderFile:
 400    """Parse decoder XML bytes, optionally resolving ``xi:include`` targets.
 401
 402    *xml_root* is the directory that ``http://jmri.org/xml/`` hrefs map
 403    to on disk — typically ``<jmri install>/xml``. When supplied,
 404    ``xi:include`` elements are resolved in-place before parsing, so
 405    variables + panes that live in shared fragments (JMRI's TCS,
 406    Digitrax, ESU decoders all do this) come through as real
 407    :class:`DecoderVariable` / :class:`DecoderPane` entries instead of
 408    silently vanishing. When *xml_root* is ``None``, xi:includes stay
 409    untouched — matches the older no-op behaviour tests rely on.
 410    """
 411    parser = etree.XMLParser(remove_blank_text=False, resolve_entities=False)
 412    root = etree.fromstring(data, parser=parser)
 413    if xml_root is not None:
 414        _resolve_xincludes(root, xml_root)
 415    return _parse_decoder_root(root, path)
 416
 417
 418def load_decoder(
 419    path: str | pathlib.Path,
 420    *,
 421    xml_root: pathlib.Path | None = None,
 422) -> DecoderFile:
 423    """Load and parse a decoder definition file.
 424
 425    When *xml_root* is omitted, it's inferred from *path* by walking
 426    upward looking for a ``decoders/`` ancestor — the standard JMRI
 427    layout is ``<xml_root>/decoders/<file>.xml``. If no ``decoders/``
 428    ancestor is found, xi:include resolution is skipped (fragment paths
 429    would be uncomputable).
 430    """
 431    p = pathlib.Path(path)
 432    if xml_root is None:
 433        xml_root = _infer_xml_root(p)
 434    return load_decoder_bytes(p.read_bytes(), p, xml_root=xml_root)
 435
 436
 437def _infer_xml_root(path: pathlib.Path) -> pathlib.Path | None:
 438    """Return the directory ``http://jmri.org/xml/`` should map to.
 439
 440    Follows JMRI's convention: decoder files live at
 441    ``<xml_root>/decoders/[<mfg>/]<file>.xml``, so the xml root is the
 442    parent of the first ancestor named ``decoders``. Returns ``None``
 443    when no such ancestor exists (e.g. when tests write decoder XML to
 444    a bare tmp dir).
 445    """
 446    for ancestor in path.parents:
 447        if ancestor.name == "decoders":
 448            return ancestor.parent
 449    return None
 450
 451
 452_JMRI_URL_PREFIX = "http://jmri.org/xml/"
 453
 454
 455def _resolve_xincludes(
 456    root: etree._Element,
 457    xml_root: pathlib.Path,
 458    _seen: set[pathlib.Path] | None = None,
 459) -> None:
 460    """Splice ``xi:include`` targets into *root*, in place.
 461
 462    JMRI's fragment convention: an included file's root element carries
 463    the same tag as the include's parent (e.g. a ``<variables>``
 464    fragment is included inside a ``<variables>`` element). In that
 465    case we splice the fragment's children in place of the
 466    ``<xi:include>``. When the tags differ (a whole ``<pane>`` fragment
 467    included at the ``<decoder-config>`` level), we substitute the
 468    fragment root itself.
 469
 470    Recurses into resolved fragments so nested includes (fragment A
 471    including fragment B) also come through. Cycle-guarded via *_seen*.
 472
 473    Unresolvable hrefs (missing file, non-JMRI URL prefix) are left as
 474    dangling ``<xi:include>`` placeholders — subsequent parsing sees
 475    no children from them, matches the "not-loaded" fallback path.
 476    """
 477    if _seen is None:
 478        _seen = set()
 479    # ``list(...)`` because we mutate the tree while iterating.
 480    for inc in list(root.iter(f"{XINCLUDE_NS}include")):
 481        href = inc.get("href", "")
 482        if not href.startswith(_JMRI_URL_PREFIX):
 483            continue
 484        rel = href[len(_JMRI_URL_PREFIX):]
 485        fragment_path = (xml_root / rel).resolve()
 486        if not fragment_path.is_file():
 487            _log.debug(
 488                "xi:include target not found: %s (from href=%s)",
 489                fragment_path, href,
 490            )
 491            continue
 492        if fragment_path in _seen:
 493            # Cycle — the fragment we're about to load is already
 494            # somewhere in the current recursion path. Bail rather
 495            # than recurse forever.
 496            _log.warning("xi:include cycle detected at %s", fragment_path)
 497            continue
 498        try:
 499            parser = etree.XMLParser(
 500                remove_blank_text=False, resolve_entities=False,
 501            )
 502            fragment_root = etree.parse(str(fragment_path), parser=parser).getroot()
 503        except (etree.XMLSyntaxError, OSError) as e:
 504            _log.debug("xi:include failed to load %s: %s", fragment_path, e)
 505            continue
 506        # Track path membership only during the descent into this
 507        # fragment. Once we return, drop it so subsequent SIBLING
 508        # xi:includes of the same fragment aren't misidentified as
 509        # cycles. Real JMRI decoder files fan out to the same enum
 510        # fragment dozens of times (each Extended Function Map row
 511        # xi:includes ECOenumFnMap.xml, etc.) and treating those as
 512        # cycles silently drops enum choices from every occurrence
 513        # past the first.
 514        _seen.add(fragment_path)
 515        try:
 516            _resolve_xincludes(fragment_root, xml_root, _seen)
 517        finally:
 518            _seen.discard(fragment_path)
 519        parent = inc.getparent()
 520        if parent is None:
 521            continue
 522        idx = list(parent).index(inc)
 523        parent.remove(inc)
 524        if fragment_root.tag == parent.tag:
 525            # Same-tag fragment → splice its children into the parent.
 526            for i, child in enumerate(list(fragment_root)):
 527                parent.insert(idx + i, child)
 528        else:
 529            # Fragment root gets substituted whole.
 530            parent.insert(idx, fragment_root)
 531
 532
 533def load_pane_bytes(data: bytes) -> DecoderPane:
 534    """Parse a standalone ``<pane>...</pane>`` fragment file.
 535
 536    JMRI ships reusable pane definitions under paths like
 537    ``xml/decoders/digitrax/PaneDigitrax.xml`` with ``<pane>`` as the
 538    root element. Decoder files pull them in via ``xi:include``.
 539    """
 540    parser = etree.XMLParser(remove_blank_text=False, resolve_entities=False)
 541    root = etree.fromstring(data, parser=parser)
 542    if root.tag != "pane":
 543        raise ValueError(f"expected <pane> root, got <{root.tag}>")
 544    return _parse_pane(root)
 545
 546
 547def load_pane(path: str | pathlib.Path) -> DecoderPane:
 548    return load_pane_bytes(pathlib.Path(path).read_bytes())
 549
 550
 551def iter_decoder_files(dir_path: str | pathlib.Path) -> Iterator[DecoderFile]:
 552    """Walk *dir_path*, yielding one :class:`DecoderFile` per ``.xml``.
 553
 554    JMRI's ``xml/decoders/`` tree contains many non-``<decoder-config>``
 555    XML files alongside the decoder definitions — NMRA CV lists, family
 556    index files, function-label fragments included via xi:include, etc.
 557    Anything whose root isn't ``<decoder-config>`` (raising ValueError
 558    from :func:`_parse_decoder_root`) or whose XML is malformed is
 559    skipped so the caller's enumeration completes across the whole tree
 560    instead of aborting on the first non-decoder file.
 561    """
 562    root_path = pathlib.Path(dir_path)
 563    for xml in sorted(root_path.rglob("*.xml")):
 564        try:
 565            yield load_decoder(xml)
 566        except etree.XMLSyntaxError as e:
 567            _log.debug("iter_decoder_files: XML parse failure %s: %s", xml, e)
 568            continue
 569        except ValueError as e:
 570            # Non-decoder XML file (index/fragment/NMRA-CVs list). These
 571            # live in the same tree by JMRI convention; just skip.
 572            _log.debug("iter_decoder_files: skipping %s (%s)", xml, e)
 573            continue
 574        except OSError as e:
 575            _log.warning("iter_decoder_files: cannot read %s: %s", xml, e)
 576            continue
 577
 578
 579def load_all_decoders(dir_path: str | pathlib.Path) -> list[DecoderFile]:
 580    return list(iter_decoder_files(dir_path))
 581
 582
 583# ----- lightweight index scan ------------------------------------------
 584#
 585# JMRI ships 2000+ decoder XMLs; parsing every variable + pane on server
 586# start is too slow for interactive use (~30-60 s on a Pi). The scanner
 587# below extracts only the identity fields we need to look a file up by
 588# (family, model) — no variables, no panes, no pydantic. Full parse is
 589# deferred until the user opens the programmer for a specific loco.
 590
 591
 592class DecoderIndexEntry(BaseModel):
 593    """One (family, model) tuple + the file it lives in.
 594
 595    ``mfg`` comes from the ``<family mfg=…>`` attribute. Both ``family``
 596    and ``mfg`` are shared across every entry from the same file; only
 597    ``model`` varies. ``source_file`` is the path to the decoder XML —
 598    caller reads it via :func:`load_decoder` when a full definition is
 599    actually needed.
 600
 601    ``low_version_id`` / ``high_version_id`` bracket which CV7 values
 602    this model accepts. Model-level ``lowVersionID``/``highVersionID``
 603    override family-level defaults; both are optional so JMRI files that
 604    don't declare them stay wildcard (0..255).
 605
 606    ``product_id`` is the ``<model productID="…"/>`` attribute — a
 607    manufacturer-specific identifier that lets identify narrow past
 608    the family. For SoundTraxx, ``productID`` is derived from CVs 253,
 609    255, and 256 (see the identify route).
 610    """
 611    model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 612
 613    family: str
 614    mfg: str
 615    model: str
 616    source_file: pathlib.Path
 617    low_version_id: int = 0
 618    high_version_id: int = 255
 619    product_id: str = ""
 620
 621
 622def iter_decoder_index(dir_path: str | pathlib.Path) -> Iterator[DecoderIndexEntry]:
 623    """Yield one :class:`DecoderIndexEntry` per (family, model) pair.
 624
 625    A single decoder XML file typically declares one family with many
 626    model rows, so one file emits multiple entries. Non-decoder XML
 627    files under the same tree (NMRA CV lists, xi:include fragments,
 628    programmer definitions) are skipped silently.
 629
 630    This function does *not* validate the file — it only touches the
 631    ``<family>`` and ``<model>`` elements. Anything else in the file
 632    (variables, panes, xi:includes) is ignored. Callers that need the
 633    full definition should call :func:`load_decoder` on the returned
 634    ``source_file`` path.
 635    """
 636    root_path = pathlib.Path(dir_path)
 637    parser = etree.XMLParser(remove_blank_text=False, resolve_entities=False)
 638    for xml in sorted(root_path.rglob("*.xml")):
 639        try:
 640            tree = etree.parse(str(xml), parser=parser)
 641        except etree.XMLSyntaxError as e:
 642            _log.debug("iter_decoder_index: XML parse failure %s: %s", xml, e)
 643            continue
 644        except OSError as e:
 645            _log.warning("iter_decoder_index: cannot read %s: %s", xml, e)
 646            continue
 647        root = tree.getroot()
 648        if root.tag != "decoder-config":
 649            # Not a decoder file (index / programmer / fragment). Skip.
 650            continue
 651        # JMRI decoder files: one <family> element under <decoder>. Use
 652        # descendant search because a few files nest differently and
 653        # xi:include can add layers.
 654        family_el = root.find(".//family")
 655        if family_el is None:
 656            continue
 657        family_name = family_el.get("name", "").strip()
 658        mfg = family_el.get("mfg", "").strip()
 659        # Family-level version range — used as the default when a
 660        # model doesn't declare its own. Missing attrs stay wildcard.
 661        def _int_or(elem: "etree._Element", attr: str, fallback: int) -> int:
 662            raw = elem.get(attr, "")
 663            try:
 664                return int(raw)
 665            except (TypeError, ValueError):
 666                return fallback
 667        fam_low = _int_or(family_el, "lowVersionID", 0)
 668        fam_high = _int_or(family_el, "highVersionID", 255)
 669        for m in family_el.findall("model"):
 670            model_name = m.get("model", "").strip()
 671            if not model_name or not family_name:
 672                # An empty family or model name would poison the index
 673                # key. Skip; log at debug so a malformed file surfaces
 674                # without spamming warnings on every load.
 675                _log.debug(
 676                    "iter_decoder_index: skipping model in %s: family=%r model=%r",
 677                    xml, family_name, model_name,
 678                )
 679                continue
 680            yield DecoderIndexEntry(
 681                family=family_name, mfg=mfg,
 682                model=model_name, source_file=xml,
 683                low_version_id=_int_or(m, "lowVersionID", fam_low),
 684                high_version_id=_int_or(m, "highVersionID", fam_high),
 685                product_id=m.get("productID", "").strip(),
 686            )
 687
 688
 689# ----- internal --------------------------------------------------------------
 690
 691
 692def _parse_decoder_root(root: etree._Element, path: pathlib.Path | None) -> DecoderFile:
 693    if root.tag != "decoder-config":
 694        raise ValueError(f"expected <decoder-config>, got <{root.tag}>")
 695
 696    decoder = root.find("decoder")
 697    if decoder is None:
 698        return DecoderFile(path=path)
 699
 700    family_elem = decoder.find("family")
 701    family = _parse_family(family_elem) if family_elem is not None else None
 702
 703    variables_elem = decoder.find("variables")
 704    variables = list(_iter_variables(variables_elem)) if variables_elem is not None else []
 705
 706    # Panes live at the ``<decoder-config>`` level, alongside ``<decoder>``.
 707    # A decoder may declare zero or more of them; standard tabs (Basic,
 708    # Motor, ...) come from the programmer definition and are loaded
 709    # separately.
 710    panes = [_parse_pane(p) for p in root.findall("pane")]
 711
 712    includes = [
 713        elem.get("href", "")
 714        for elem in root.iter(f"{XINCLUDE_NS}include")
 715        if elem.get("href")
 716    ]
 717
 718    return DecoderFile(
 719        path=path, family=family, variables=variables,
 720        panes=panes, includes=includes,
 721    )
 722
 723
 724def _parse_pane(elem: etree._Element) -> DecoderPane:
 725    """Turn a ``<pane>`` element into a :class:`DecoderPane`.
 726
 727    JMRI's pane structure is nested (``<column>`` / ``<row>`` containing
 728    ``<display>`` / ``<label>`` / ``<separator>`` / ``<cvtable>`` /
 729    ``<dccaddress>`` / further ``<column>``/``<row>``), but for our
 730    change-tracking + queue-scoping purposes we care about the flat
 731    list of ``<display item="...">`` references. ``elem.iter()`` walks
 732    the whole subtree, giving us that flat list in document order.
 733    Rich layout rendering (step 3) works off the preserved
 734    ``raw_element``.
 735    """
 736    default_name = ""
 737    name_by_lang: dict[str, str] = {}
 738    for name_elem in elem.findall("name"):
 739        text = (name_elem.text or "").strip()
 740        lang = name_elem.get(XML_LANG_ATTR, "")
 741        if lang:
 742            name_by_lang[lang] = text
 743        elif not default_name:
 744            default_name = text
 745
 746    items = [
 747        d.get("item", "")
 748        for d in elem.iter("display")
 749        if d.get("item")
 750    ]
 751    has_cvtable = elem.find(".//cvtable") is not None
 752    has_dcc_address = elem.find(".//dccaddress") is not None
 753    has_fnmapping = elem.find(".//fnmapping") is not None
 754    # JMRI's Speed Table pane widget: catch both the standard
 755    # ``<speedtable/>`` element and any case variant / adjacent widget
 756    # whose tag name starts with "speedtable" (JMRI has shipped both
 757    # ``<speedtable/>`` and ``<speedTable/>`` across versions, plus
 758    # helper widgets like ``<speedTableVarSlider/>``). Whenever any of
 759    # those is present, the render layer expands the decoder's
 760    # SPEED_TABLE variable into per-step CV rows.
 761    has_speed_table = any(
 762        isinstance(d.tag, str) and d.tag.lower().startswith("speedtable")
 763        for d in elem.iter()
 764    )
 765
 766    return DecoderPane(
 767        name=default_name,
 768        name_by_lang=name_by_lang,
 769        items=items,
 770        has_cvtable=has_cvtable,
 771        has_dcc_address=has_dcc_address,
 772        has_speed_table=has_speed_table,
 773        has_fnmapping=has_fnmapping,
 774        raw_element=elem,
 775    )
 776
 777
 778# ----- rich pane layout tree --------------------------------------------
 779
 780
 781# Container elements — nest child nodes.
 782_LAYOUT_CONTAINER_TAGS = {"column", "row", "group"}
 783# Widget leaf elements the renderer knows how to draw.
 784_LAYOUT_LEAF_TAGS = {
 785    "display", "label", "separator",
 786    "cvtable", "speedtable", "dccaddress", "fnmapping",
 787}
 788# Grid layout — ``<grid>`` wraps ``<griditem gridx="…" gridy="…">``
 789# children, each of which is itself a container of layout nodes.
 790_LAYOUT_GRID_TAG = "grid"
 791_LAYOUT_GRIDITEM_TAG = "griditem"
 792
 793
 794def pane_layout_tree(pane: "DecoderPane") -> dict:
 795    """Build a nested tree describing the pane's authored layout.
 796
 797    Walks :attr:`DecoderPane.raw_element` and emits a tree of dicts
 798    the renderer can iterate recursively. Recognises JMRI's usual
 799    layout primitives — ``<column>``, ``<row>``, ``<group>``,
 800    ``<display>``, ``<label>``, ``<separator>``, ``<cvtable>``,
 801    ``<speedtable>``, ``<dccaddress>``.
 802
 803    Nodes carry a ``type`` key and either a ``children`` list
 804    (containers) or type-specific attributes (leaves):
 805
 806    * ``{"type": "column", "children": [...]}``
 807    * ``{"type": "display", "item": "Vstart"}``
 808    * ``{"type": "label", "text": "Speed control"}``
 809    * ``{"type": "separator"}``
 810
 811    Any element the renderer doesn't recognise (rare — hand-authored
 812    decoder files sometimes have quirks) is silently skipped, keeping
 813    the rest of the tree renderable.
 814
 815    When *pane* has no ``raw_element`` (e.g. constructed in a test
 816    from a bare ``DecoderPane(name="X", items=[…])``), the returned
 817    tree is a flat column of the pane's items — matches what the
 818    legacy flat-table renderer showed and keeps tests uncoupled from
 819    XML shape.
 820    """
 821    if pane.raw_element is None:
 822        return _dedupe_display_nodes({
 823            "type": "column",
 824            "children": [
 825                {"type": "display", "item": item}
 826                for item in pane.items
 827            ],
 828        })
 829    root_children = list(_layout_children(pane.raw_element))
 830    # If the pane has no explicit container at the top level, wrap in
 831    # a column so the renderer always starts from a container node.
 832    if len(root_children) == 1 and root_children[0]["type"] in _LAYOUT_CONTAINER_TAGS:
 833        return _dedupe_display_nodes(root_children[0])
 834    return _dedupe_display_nodes(
 835        {"type": "column", "children": root_children},
 836    )
 837
 838
 839def _dedupe_display_nodes(tree: dict) -> dict:
 840    """Walk *tree* and drop repeat ``<display item="X">`` leaves.
 841
 842    JMRI's pane XML sometimes references the same variable twice —
 843    typically for two widget presentations (numeric input + slider),
 844    or because two ``<column>`` blocks each pull it in. Our renderer
 845    only knows how to draw one widget per variable, so a naive walk
 846    produces duplicate rows side-by-side in the same column, or the
 847    same variable in two adjacent columns. Track seen items across
 848    the WHOLE tree (not just per container) and drop later
 849    occurrences.
 850
 851    Mutates the tree in place and returns it for chaining.
 852    """
 853    seen: set[str] = set()
 854
 855    def prune(node: dict) -> dict | None:
 856        if node.get("type") == "display":
 857            item = node.get("item", "")
 858            if not item or item in seen:
 859                return None
 860            seen.add(item)
 861            return node
 862        if "children" in node:
 863            pruned: list[dict] = []
 864            for child in node["children"]:
 865                kept = prune(child)
 866                if kept is not None:
 867                    pruned.append(kept)
 868            node["children"] = pruned
 869        return node
 870
 871    prune(tree)
 872    return tree
 873
 874
 875def _layout_children(elem: "etree._Element") -> Iterator[dict]:
 876    """Iterate the layout-relevant children of *elem* as tree nodes."""
 877    for child in elem:
 878        if not isinstance(child.tag, str):
 879            # Comments, processing instructions — skip.
 880            continue
 881        node = _layout_node(child)
 882        if node is not None:
 883            yield node
 884
 885
 886def _layout_griditem(elem: "etree._Element") -> dict | None:
 887    """Turn a ``<griditem>`` into a placement + children node.
 888
 889    JMRI grid items use Swing's ``GridBagLayout`` attribute names.
 890    We preserve the ones the renderer needs and drop the rest:
 891
 892    * ``gridx`` / ``gridy`` — 0-based cell coordinates.
 893    * ``gridwidth`` / ``gridheight`` — spans. JMRI also uses the
 894      string ``REMAINDER`` for "span to the end of the row/column".
 895    * ``anchor`` — Swing anchor constant (``LINE_END``, ``CENTER``,
 896      etc.); we map the common ones to CSS text-alignment.
 897    """
 898    if not isinstance(elem.tag, str) or elem.tag != _LAYOUT_GRIDITEM_TAG:
 899        return None
 900    def _int_attr(name: str) -> int | None:
 901        raw = elem.get(name)
 902        if raw is None or raw == "":
 903            return None
 904        try:
 905            return int(raw)
 906        except ValueError:
 907            return None
 908    return {
 909        "type": "griditem",
 910        "gridx": _int_attr("gridx"),
 911        "gridy": _int_attr("gridy"),
 912        "gridwidth": elem.get("gridwidth") or "",
 913        "gridheight": elem.get("gridheight") or "",
 914        "anchor": elem.get("anchor") or "",
 915        "children": list(_layout_children(elem)),
 916    }
 917
 918
 919def _layout_node(elem: "etree._Element") -> dict | None:
 920    """Turn one XML element into a layout tree node, or None to skip.
 921
 922    Ordering matters for the renderer, so we preserve document order
 923    and only drop elements the renderer wouldn't know what to do with
 924    (rare metadata like ``<name>``, ``<qualifier>`` — those are
 925    consumed at parse time or belong elsewhere).
 926    """
 927    tag = elem.tag
 928    if not isinstance(tag, str):
 929        return None
 930    if tag in _LAYOUT_CONTAINER_TAGS:
 931        return {
 932            "type": tag,
 933            "children": list(_layout_children(elem)),
 934        }
 935    if tag == _LAYOUT_GRID_TAG:
 936        return {
 937            "type": "grid",
 938            "children": [
 939                node for node in (_layout_griditem(c) for c in elem)
 940                if node is not None
 941            ],
 942        }
 943    if tag == "display":
 944        item = elem.get("item", "")
 945        if not item:
 946            return None
 947        return {
 948            "type": "display",
 949            "item": item,
 950            # ``format`` decides how the widget renders:
 951            # "checkbox" collapses an int variable to a bool checkbox,
 952            # "" / absent means the variable's natural widget (enum
 953            # dropdown, numeric input, …). Preserving the raw attribute
 954            # here lets the template pick without another lookup.
 955            "format": elem.get("format", ""),
 956            # ``label=""`` on a display suppresses the widget's own
 957            # label — used inside grid cells where the label is in a
 958            # neighbouring cell instead.
 959            "label_override": elem.get("label"),
 960        }
 961    if tag == "label":
 962        # ``<label>text</label>`` or ``<label><text>text</text></label>``.
 963        # Localised variants match the same rule as variable labels.
 964        text = _pick_localized_text(elem, "text") or (elem.text or "").strip()
 965        return {"type": "label", "text": text}
 966    if tag == "separator":
 967        return {"type": "separator"}
 968    if tag == "cvtable":
 969        return {"type": "cvtable"}
 970    if tag == "dccaddress":
 971        return {"type": "dccaddress"}
 972    if tag == "fnmapping":
 973        return {"type": "fnmapping"}
 974    # ``<speedtable/>`` and any case/tag variant that starts with
 975    # "speedtable" (older JMRI files shipped ``<speedTable>``,
 976    # ``<speedTableVarSlider>``, etc.).
 977    if tag.lower().startswith("speedtable"):
 978        return {"type": "speedtable"}
 979    return None
 980
 981
 982def _parse_family(elem: etree._Element) -> DecoderFamily:
 983    models = [_parse_model(m) for m in elem.findall("model")]
 984    outputs = [
 985        DecoderOutput(name=o.get("name", ""), label=o.get("label", ""))
 986        for o in elem.findall("output")
 987    ]
 988    return DecoderFamily(
 989        name=elem.get("name", ""),
 990        mfg=elem.get("mfg", ""),
 991        models=models,
 992        outputs=outputs,
 993    )
 994
 995
 996def _parse_model(elem: etree._Element) -> DecoderModel:
 997    outputs = [
 998        DecoderOutput(name=o.get("name", ""), label=o.get("label", ""))
 999        for o in elem.findall("output")
1000    ]
1001    protocols = [
1002        (p.text or "").strip()
1003        for p in elem.findall("protocols/protocol")
1004        if (p.text or "").strip()
1005    ]
1006    size = elem.find("size")
1007    return DecoderModel(
1008        model=elem.get("model", ""),
1009        connector=elem.get("connector", ""),
1010        show_empty_panes=elem.get("showEmptyPanes"),
1011        max_input_volts=elem.get("maxInputVolts"),
1012        max_motor_current=elem.get("maxMotorCurrent"),
1013        outputs=outputs,
1014        protocols=protocols,
1015        size_length=size.get("length") if size is not None else None,
1016        size_width=size.get("width") if size is not None else None,
1017        size_height=size.get("height") if size is not None else None,
1018        size_units=size.get("units") if size is not None else None,
1019        product_id=elem.get("productID", ""),
1020    )
1021
1022
1023def _parse_default_int(raw: str | None) -> int | None:
1024    """Coerce a ``<variable default="…">`` string to an int.
1025
1026    Real JMRI decoder files use plain decimal ints for the ``default``
1027    attribute. A handful of hand-authored files use ``0x…`` hex — accept
1028    that too via ``base=0`` (auto-detects). Anything else (blank,
1029    missing, malformed) returns ``None`` so downstream can use its
1030    "no known default" fallback path.
1031    """
1032    if raw is None:
1033        return None
1034    raw = raw.strip()
1035    if not raw:
1036        return None
1037    try:
1038        return int(raw, 0)
1039    except (TypeError, ValueError):
1040        return None
1041
1042
1043def _pick_localized_text(
1044    elem: etree._Element, tag: str, preferred_lang: str = "en",
1045) -> str:
1046    """Pick the best localized text out of ``<tag>`` children.
1047
1048    JMRI decoder XMLs sometimes carry the same descriptive text in
1049    multiple languages via ``<label xml:lang="…">…</label>`` — the
1050    same variable can have an English, Italian, French, German, and
1051    Dutch label. The user's UI is English; we shouldn't silently render
1052    the Italian one just because ``elem.find(tag)`` happened to hit it
1053    first.
1054
1055    Priority order:
1056
1057    1. The ``<tag>`` element with ``xml:lang="<preferred_lang>"``.
1058    2. The ``<tag>`` element with NO ``xml:lang`` attribute (JMRI's
1059       convention: the unqualified element is the default English
1060       text — same rule the ``<pane><name>`` parser uses).
1061    3. Any other ``<tag>`` element (last-resort fallback).
1062    4. Empty string.
1063
1064    Preserves inline markup like ``<br/>`` and ``<b>``: JMRI tooltips
1065    routinely contain such tags for line breaks and emphasis. We serialize
1066    the element's content back to a string, stripping only the wrapping
1067    ``<tag>…</tag>`` — the caller can render it as HTML.
1068    """
1069    children = elem.findall(tag)
1070    if not children:
1071        return ""
1072
1073    def _inner_html(node: etree._Element) -> str:
1074        out = node.text or ""
1075        for child in node:
1076            out += etree.tostring(child, encoding="unicode")
1077        return out.strip()
1078
1079    preferred = None
1080    unqualified = None
1081    fallback = None
1082    for c in children:
1083        lang = c.get(XML_LANG_ATTR, "")
1084        if lang == preferred_lang and preferred is None:
1085            preferred = c
1086        elif not lang and unqualified is None:
1087            unqualified = c
1088        elif fallback is None:
1089            fallback = c
1090    # lxml Elements are falsy when they have no children — use
1091    # explicit ``is not None`` checks so we don't accidentally skip an
1092    # empty ``<label></label>``.
1093    picked = preferred
1094    if picked is None:
1095        picked = unqualified
1096    if picked is None:
1097        picked = fallback
1098    if picked is None:
1099        picked = children[0]
1100    return _inner_html(picked)
1101
1102
1103def _iter_variables(elem: etree._Element) -> Iterable[DecoderVariable]:
1104    """Yield CV variables, skipping XInclude pointers (captured separately).
1105
1106    Populates the step-2 typed fields (``kind`` + type-specific structure)
1107    based on which value-type child element the variable declares. If
1108    none is present (rare — usually a placeholder), ``kind`` stays
1109    :data:`VariableKind.UNKNOWN` and the variable acts as an opaque byte.
1110    """
1111    for v in elem.findall("variable"):
1112        label_text = _pick_localized_text(v, "label")
1113        tooltip_text = _pick_localized_text(v, "tooltip")
1114        # ``<qualifier>`` (or ``<qualifiers><qualifier>``): runtime
1115        # visibility gates. Each holds a ``variableref`` + ``relation`` +
1116        # ``value`` triple. Collect them flat so the caller can AND them.
1117        qualifier_elems = list(v.findall("qualifier"))
1118        qualifier_elems.extend(v.findall("qualifiers/qualifier"))
1119        qualifier_dicts: list[dict[str, str]] = []
1120        for q in qualifier_elems:
1121            vref = q.find("variableref")
1122            rel = q.find("relation")
1123            val = q.find("value")
1124            if vref is None or rel is None:
1125                continue
1126            qualifier_dicts.append({
1127                "variableref": (vref.text or "").strip(),
1128                "relation": (rel.text or "").strip(),
1129                "value": (val.text or "").strip() if val is not None else "",
1130            })
1131        # Mask may be on <variable> directly or inside <splitVal>.
1132        split_elem = _first_child(v, ("splitVal", "splitHexVal", "splitDateTimeVal"))
1133        mask = v.get("mask") or (split_elem.get("upperMask") if split_elem is not None else None)
1134        min_val = v.get("min") or (split_elem.get("min") if split_elem is not None else None)
1135        max_val = v.get("max") or (split_elem.get("max") if split_elem is not None else None)
1136
1137        cv_attr = v.get("CV", "")
1138        kind, kind_extras = _classify_value_type(v)
1139
1140        # Populate ``split`` for ANY variable whose CV attribute uses
1141        # dotted notation (``16.1.275`` = indexed CV, ``275:4`` =
1142        # 4-byte split, ``17.18`` = long-address pair). The value-type
1143        # child (decVal / hexVal / enumVal / …) doesn't necessarily
1144        # signal indexed access — a plain ``<decVal>`` on
1145        # ``CV="16.1.275"`` is a decimal variable that happens to sit
1146        # at bank/index 16/1, base CV 275. Without this fallback, the
1147        # write path would send the byte to CV 275 under the current
1148        # (uncontrolled) bank/index — corrupting unrelated state.
1149        parsed_split = kind_extras.get("split")
1150        if parsed_split is None and cv_attr and (
1151            "." in cv_attr or ":" in cv_attr
1152        ):
1153            parsed_split = SplitCv.parse(cv_attr)
1154        yield DecoderVariable(
1155            CV=cv_attr,
1156            item=v.get("item", ""),
1157            default=_parse_default_int(v.get("default")),
1158            mask=mask,
1159            min=min_val,
1160            max=max_val,
1161            label=label_text,
1162            ops_only=(v.get("opsOnly", "no").lower() == "yes"),
1163            kind=kind,
1164            bit_mask=CvBitMask.parse(mask),
1165            enum_choices=kind_extras.get("enum_choices", []),
1166            composite_choices=kind_extras.get("composite_choices", []),
1167            split=parsed_split,
1168            constant_value=kind_extras.get("constant_value"),
1169            speed_table_entries=kind_extras.get("speed_table_entries", 28),
1170            read_only=(v.get("readOnly", "no").lower() == "yes"),
1171            write_only=(v.get("writeOnly", "no").lower() == "yes"),
1172            tooltip=tooltip_text or v.get("tooltip", ""),
1173            include_product_ids=[
1174                p.strip() for p in v.get("include", "").split(",")
1175                if p.strip()
1176            ],
1177            exclude_product_ids=[
1178                p.strip() for p in v.get("exclude", "").split(",")
1179                if p.strip()
1180            ],
1181            qualifiers=qualifier_dicts,
1182        )
1183
1184
1185def _first_child(elem: etree._Element, tags: tuple[str, ...]) -> etree._Element | None:
1186    for tag in tags:
1187        child = elem.find(tag)
1188        if child is not None:
1189            return child
1190    return None
1191
1192
1193# Map value-type child element tag → VariableKind.
1194_KIND_BY_TAG = {
1195    "decVal":            VariableKind.DECIMAL,
1196    "hexVal":            VariableKind.HEXADECIMAL,
1197    "enumVal":           VariableKind.ENUM,
1198    "speedTable":        VariableKind.SPEED_TABLE,
1199    "shortAddressVal":   VariableKind.SHORT_ADDRESS,
1200    "longAddressVal":    VariableKind.LONG_ADDRESS,
1201    "splitVal":          VariableKind.SPLIT,
1202    "splitHexVal":       VariableKind.SPLIT_HEX,
1203    "splitDateTimeVal":  VariableKind.SPLIT_DATETIME,
1204    "compositeVal":      VariableKind.COMPOSITE,
1205    "constant":          VariableKind.CONSTANT,
1206}
1207
1208
1209def _classify_value_type(v: etree._Element) -> tuple[VariableKind, dict]:
1210    """Look at the value-type children of ``<variable>`` and pick a kind.
1211
1212    Returns ``(kind, extras)`` where ``extras`` is a dict of type-specific
1213    parsed structure the caller drops into the :class:`DecoderVariable`
1214    constructor.
1215    """
1216    cv_attr = v.get("CV", "")
1217    for tag, kind in _KIND_BY_TAG.items():
1218        child = v.find(tag)
1219        if child is None:
1220            continue
1221        if kind is VariableKind.ENUM:
1222            return kind, {"enum_choices": list(_iter_enum_choices(child))}
1223        if kind is VariableKind.COMPOSITE:
1224            return kind, {"composite_choices": list(_iter_composite_choices(child))}
1225        if kind in (
1226            VariableKind.SPLIT, VariableKind.SPLIT_HEX, VariableKind.SPLIT_DATETIME,
1227        ):
1228            return kind, {"split": SplitCv.parse(cv_attr)}
1229        if kind is VariableKind.LONG_ADDRESS:
1230            # CV17.CV18 pair. If the caller wrote CV="17", we still
1231            # need to know we're touching CV18 too.
1232            split_cv = SplitCv.parse(cv_attr or "17.18")
1233            return kind, {"split": split_cv}
1234        if kind is VariableKind.CONSTANT:
1235            raw = child.get("value")
1236            try:
1237                cval = int(raw) if raw is not None else None
1238            except ValueError:
1239                cval = None
1240            return kind, {"constant_value": cval}
1241        if kind is VariableKind.SPEED_TABLE:
1242            # ``<speedTable entries="N"/>`` declares the number of
1243            # speed-step slots; NMRA default is 28 (CV67..CV94).
1244            raw = child.get("entries")
1245            try:
1246                entries = int(raw) if raw else 28
1247            except ValueError:
1248                entries = 28
1249            return kind, {"speed_table_entries": max(entries, 1)}
1250        # decVal / hexVal / shortAddressVal — no extras.
1251        return kind, {}
1252    return VariableKind.UNKNOWN, {}
1253
1254
1255def _iter_enum_choices(elem: etree._Element) -> Iterator[EnumChoice]:
1256    """Walk ``<enumVal>`` yielding one :class:`EnumChoice` per
1257    ``<enumChoice>``. Positional index becomes the wire value when
1258    the element has no explicit ``value`` attribute.
1259    """
1260    for index, ec in enumerate(elem.findall("enumChoice")):
1261        raw = ec.get("value")
1262        try:
1263            value = int(raw) if raw is not None else index
1264        except ValueError:
1265            value = index
1266        # Default choice text is the ``choice`` attribute; localised
1267        # variants appear as <choice xml:lang="…">…</choice> children.
1268        default = ec.get("choice", "")
1269        by_lang: dict[str, str] = {}
1270        for c in ec.findall("choice"):
1271            text = (c.text or "").strip()
1272            lang = c.get(XML_LANG_ATTR, "")
1273            if lang:
1274                by_lang[lang] = text
1275            elif not default and text:
1276                default = text
1277        yield EnumChoice(value=value, choice=default, choice_by_lang=by_lang)
1278
1279
1280def _iter_composite_choices(elem: etree._Element) -> Iterator[CompositeChoice]:
1281    """Walk ``<compositeVal>`` yielding one :class:`CompositeChoice` per
1282    ``<compositeChoice>`` plus its ``<compositeSetting>`` rows."""
1283    for cc in elem.findall("compositeChoice"):
1284        default = cc.get("choice", "")
1285        by_lang: dict[str, str] = {}
1286        for c in cc.findall("choice"):
1287            text = (c.text or "").strip()
1288            lang = c.get(XML_LANG_ATTR, "")
1289            if lang:
1290                by_lang[lang] = text
1291            elif not default and text:
1292                default = text
1293        settings = []
1294        for cs in cc.findall("compositeSetting"):
1295            try:
1296                value = int(cs.get("value", "0"))
1297            except ValueError:
1298                value = 0
1299            settings.append(
1300                CompositeSetting(label=cs.get("label", ""), value=value)
1301            )
1302        yield CompositeChoice(
1303            choice=default,
1304            choice_by_lang=by_lang,
1305            settings=settings,
1306        )
1307
1308
1309# ----- variable encode / decode ------------------------------------------
1310
1311
1312class UnsupportedVariableKind(ValueError):
1313    """Raised when a caller tries to encode/decode a variable whose kind
1314    doesn't have a byte-level round-trip yet (COMPOSITE fans out to
1315    sub-variables; SPEED_TABLE has custom widget semantics)."""
1316
1317
1318def variable_primary_cv(var: DecoderVariable) -> int:
1319    """The primary CV byte this variable reads/writes.
1320
1321    For split variables the primary CV is the low-address byte of the
1322    split. For simple variables it's just ``int(var.cv)``. Returns 0
1323    if the CV attribute isn't a plain integer we can parse — the
1324    caller should treat that as "not directly programmable".
1325    """
1326    if var.split is not None:
1327        return var.split.primary_cv
1328    try:
1329        return int(var.cv)
1330    except ValueError:
1331        return 0
1332
1333
1334def variable_cv_list(var: DecoderVariable) -> list[int]:
1335    """Every CV byte this variable touches, low-address first.
1336
1337    * Single-byte variables → ``[primary_cv]``
1338    * Long-address → ``[17, 18]``
1339    * Split-CV — depends on ``byte_count`` on the parsed :class:`SplitCv`.
1340    """
1341    if var.kind is VariableKind.LONG_ADDRESS:
1342        return [17, 18]
1343    if var.split is not None:
1344        return var.split.cv_list
1345    primary = variable_primary_cv(var)
1346    return [primary] if primary else []
1347
1348
1349def variable_is_indexed(var: DecoderVariable) -> bool:
1350    """True when *var* uses ``bank.index.base`` indexed CV addressing.
1351
1352    Indexed access (Tsunami2 / ESU / newer NCE decoders) requires
1353    writing ``CV31=bank`` + ``CV32=index`` before every read or write
1354    to the base CV. The decoder maintains bank/index state internally
1355    and interprets subsequent CV257–CV511 accesses through that page.
1356    """
1357    return var.split is not None and var.split.bank is not None
1358
1359
1360def variable_store_key(var: DecoderVariable, cv_number: int) -> str:
1361    """Return the roster ``<CVvalue name="…">`` key for *cv_number* under *var*.
1362
1363    JMRI's roster file writes each CV under a distinct key:
1364
1365    * Plain CV: ``"275"``
1366    * Indexed CV: ``"16.1.275"`` — bank + index + base
1367
1368    So `16.1.275` and `275` coexist without colliding — they mean
1369    different CVs on the wire (same base but different decoder pages).
1370    """
1371    if variable_is_indexed(var):
1372        return f"{var.split.bank}.{var.split.index or 0}.{cv_number}"
1373    return str(cv_number)
1374
1375
1376@dataclass(frozen=True)
1377class VariableWriteOp:
1378    """One physical CV write generated by :func:`variable_wire_write_plan`.
1379
1380    * ``wire_cv`` — actual CV number sent on the wire (bare integer;
1381      the DCC packet targets this CV under whatever bank/index the
1382      decoder currently has).
1383    * ``byte_value`` — 0..255 byte to write.
1384    * ``store_key`` — key to persist under in the roster's cv_values
1385      dict, or ``None`` when the op is bank/index setup (CV31/CV32)
1386      that shouldn't leave a per-write trace.
1387    """
1388    wire_cv: int
1389    byte_value: int
1390    store_key: str | None
1391
1392
1393def variable_wire_write_plan(
1394    var: DecoderVariable,
1395    human_value: int,
1396    cv_values: dict[str, int],
1397) -> list[VariableWriteOp]:
1398    """Ordered wire writes to set *var* to *human_value*.
1399
1400    Indexed variables produce three-or-more ops: ``CV31=bank``,
1401    ``CV32=index``, then one or more base-CV writes. The setup ops
1402    carry ``store_key=None`` so callers can skip persisting them
1403    (they'd corrupt the roster's per-variable tracking otherwise).
1404
1405    Non-indexed variables produce the same wire writes as
1406    :func:`variable_apply_to_state` but wrapped in :class:`VariableWriteOp`
1407    with plain string keys (``"275"``) for round-trip storage.
1408
1409    Read-only / CONSTANT variables return an empty list.
1410    """
1411    plain = variable_apply_to_state(var, human_value, _int_keyed(cv_values))
1412    if not plain:
1413        return []
1414    ops: list[VariableWriteOp] = []
1415    if variable_is_indexed(var):
1416        assert var.split is not None  # narrow for the type checker
1417        ops.append(VariableWriteOp(wire_cv=31, byte_value=var.split.bank or 0, store_key=None))
1418        ops.append(VariableWriteOp(wire_cv=32, byte_value=var.split.index or 0, store_key=None))
1419    for wire_cv, byte in plain.items():
1420        ops.append(VariableWriteOp(
1421            wire_cv=wire_cv, byte_value=byte,
1422            store_key=variable_store_key(var, wire_cv),
1423        ))
1424    return ops
1425
1426
1427def variable_wire_read_plan(var: DecoderVariable) -> list[VariableWriteOp]:
1428    """Ordered wire ops to read *var*'s current bytes.
1429
1430    For plain variables: one entry per CV in :func:`variable_cv_list`.
1431    For indexed variables: ``CV31=bank`` + ``CV32=index`` setup ops
1432    (as writes with ``store_key=None``) then one read entry per base
1433    CV. Callers should send the setup writes and then read the
1434    ``wire_cv`` of any op whose ``store_key`` is non-None.
1435    """
1436    if variable_is_indexed(var):
1437        assert var.split is not None
1438        ops: list[VariableWriteOp] = [
1439            VariableWriteOp(wire_cv=31, byte_value=var.split.bank or 0, store_key=None),
1440            VariableWriteOp(wire_cv=32, byte_value=var.split.index or 0, store_key=None),
1441        ]
1442        for cv in variable_cv_list(var):
1443            ops.append(VariableWriteOp(
1444                wire_cv=cv, byte_value=0,   # unused for reads
1445                store_key=variable_store_key(var, cv),
1446            ))
1447        return ops
1448    # Non-indexed: no setup, just read each CV.
1449    return [
1450        VariableWriteOp(wire_cv=cv, byte_value=0, store_key=str(cv))
1451        for cv in variable_cv_list(var)
1452    ]
1453
1454
1455def _int_keyed(cv_values: dict[str, int]) -> dict[int, int]:
1456    """Convert a string-keyed CV snapshot to int-keyed for legacy helpers.
1457
1458    Silently drops keys that aren't plain integers — indexed keys
1459    like ``"16.1.275"`` don't participate in bit-mask RMW anyway;
1460    those are handled per-byte by the write plan directly.
1461    """
1462    out: dict[int, int] = {}
1463    for k, v in cv_values.items():
1464        try:
1465            out[int(k)] = int(v)
1466        except (TypeError, ValueError):
1467            continue
1468    return out
1469
1470
1471def variable_extract_from_store(
1472    var: DecoderVariable, cv_values: dict[str, int],
1473) -> int:
1474    """Like :func:`variable_extract_from_state` but keyed by store keys.
1475
1476    * Plain variables look up ``str(primary_cv)``.
1477    * Indexed variables look up ``"{bank}.{index}.{cv}"`` — the same
1478      shape JMRI Java writes in the roster's ``<CVvalue name="…">``.
1479    * Multi-byte splits assemble big-endian from consecutive keys.
1480
1481    Returns 0 when a key is missing (the caller should overlay a
1482    catalog default separately).
1483    """
1484    if var.kind is VariableKind.CONSTANT:
1485        return var.constant_value or 0
1486    if var.kind is VariableKind.LONG_ADDRESS:
1487        high = int(cv_values.get("17", 0)) & 0x3F
1488        low = int(cv_values.get("18", 0)) & 0xFF
1489        return (high << 8) | low
1490    if var.kind in (VariableKind.SPLIT, VariableKind.SPLIT_HEX):
1491        acc = 0
1492        for cv in variable_cv_list(var):
1493            key = variable_store_key(var, cv)
1494            acc = (acc << 8) | (int(cv_values.get(key, 0)) & 0xFF)
1495        return acc
1496    if var.kind in (
1497        VariableKind.DECIMAL, VariableKind.HEXADECIMAL,
1498        VariableKind.ENUM, VariableKind.UNKNOWN, VariableKind.SHORT_ADDRESS,
1499    ):
1500        key = variable_store_key(var, variable_primary_cv(var))
1501        byte = int(cv_values.get(key, 0))
1502        return var.bit_mask.extract(byte)
1503    raise UnsupportedVariableKind(
1504        f"variable_extract_from_store does not support {var.kind}"
1505    )
1506
1507
1508def variable_extract_from_state(
1509    var: DecoderVariable, cv_bytes: dict[int, int],
1510) -> int:
1511    """Decode the variable's *human value* from a snapshot of CV bytes.
1512
1513    * DECIMAL / HEXADECIMAL — the masked byte value
1514    * ENUM — the wire value (caller can look up the choice label)
1515    * SHORT_ADDRESS — CV1 straight through
1516    * LONG_ADDRESS — ``(CV17 & 0x3F) << 8 | CV18`` (strip the two long-
1517      address marker bits set on CV17)
1518    * SPLIT / SPLIT_HEX — big-endian assembly across ``cv_list``
1519    * CONSTANT — the pinned constant value
1520    * SPLIT_DATETIME, COMPOSITE, SPEED_TABLE — raise
1521      :class:`UnsupportedVariableKind`; those need step-specific work.
1522    """
1523    if var.kind is VariableKind.CONSTANT:
1524        return var.constant_value or 0
1525    if var.kind is VariableKind.SHORT_ADDRESS:
1526        return cv_bytes.get(variable_primary_cv(var), 0)
1527    if var.kind is VariableKind.LONG_ADDRESS:
1528        high = cv_bytes.get(17, 0) & 0x3F
1529        low = cv_bytes.get(18, 0) & 0xFF
1530        return (high << 8) | low
1531    if var.kind in (VariableKind.SPLIT, VariableKind.SPLIT_HEX):
1532        acc = 0
1533        for cv in variable_cv_list(var):
1534            acc = (acc << 8) | (cv_bytes.get(cv, 0) & 0xFF)
1535        return acc
1536    if var.kind in (
1537        VariableKind.DECIMAL, VariableKind.HEXADECIMAL,
1538        VariableKind.ENUM, VariableKind.UNKNOWN,
1539    ):
1540        byte = cv_bytes.get(variable_primary_cv(var), 0)
1541        return var.bit_mask.extract(byte)
1542    raise UnsupportedVariableKind(
1543        f"variable_extract_from_state does not support {var.kind}"
1544    )
1545
1546
1547def variable_apply_to_state(
1548    var: DecoderVariable, human_value: int, cv_bytes: dict[int, int],
1549) -> dict[int, int]:
1550    """Compute the CV-byte writes needed to set the variable to *human_value*.
1551
1552    Returns ``{cv: new_byte_value}`` for every CV the write touches.
1553    Masked variables read the current byte out of *cv_bytes* to preserve
1554    the non-``V`` bits (read-modify-write); *cv_bytes* is the caller's
1555    running snapshot of committed values.
1556    """
1557    if var.kind is VariableKind.CONSTANT:
1558        return {}     # constants never write
1559    if var.read_only:
1560        return {}
1561    if var.kind is VariableKind.SHORT_ADDRESS:
1562        primary = variable_primary_cv(var)
1563        return {primary: human_value & 0x7F}
1564    if var.kind is VariableKind.LONG_ADDRESS:
1565        # NMRA: high byte carries the 0b11000000 long-address marker.
1566        addr = int(human_value) & 0x3FFF
1567        return {17: 0xC0 | ((addr >> 8) & 0x3F), 18: addr & 0xFF}
1568    if var.kind in (VariableKind.SPLIT, VariableKind.SPLIT_HEX):
1569        cvs = variable_cv_list(var)
1570        n = len(cvs)
1571        writes: dict[int, int] = {}
1572        v = int(human_value)
1573        # High-byte first, matching how ``extract`` reassembles.
1574        for idx, cv in enumerate(cvs):
1575            shift = 8 * (n - 1 - idx)
1576            writes[cv] = (v >> shift) & 0xFF
1577        return writes
1578    if var.kind in (
1579        VariableKind.DECIMAL, VariableKind.HEXADECIMAL,
1580        VariableKind.ENUM, VariableKind.UNKNOWN,
1581    ):
1582        primary = variable_primary_cv(var)
1583        current = cv_bytes.get(primary, 0)
1584        return {primary: var.bit_mask.apply(current, int(human_value))}
1585    raise UnsupportedVariableKind(
1586        f"variable_apply_to_state does not support {var.kind}"
1587    )
1588
1589
1590def resolve_composite_choice(
1591    var: DecoderVariable, choice_text: str,
1592) -> list[CompositeSetting]:
1593    """Return the ordered ``<compositeSetting>`` list for the named choice.
1594
1595    Composite variables fan out to sub-variable writes: picking a choice
1596    means writing each of its settings to the referenced sub-variables
1597    (looked up by ``item`` name). This helper does the choice → settings
1598    lookup; the caller (session/queue layer in step 4) is responsible
1599    for finding the referenced variables and applying their writes.
1600
1601    Raises ``KeyError`` if the choice text isn't a declared option.
1602    """
1603    if var.kind is not VariableKind.COMPOSITE:
1604        raise UnsupportedVariableKind(
1605            f"resolve_composite_choice: variable is {var.kind}, not COMPOSITE"
1606        )
1607    for cc in var.composite_choices:
1608        if cc.choice == choice_text:
1609            return list(cc.settings)
1610        if choice_text in cc.choice_by_lang.values():
1611            return list(cc.settings)
1612    raise KeyError(
1613        f"no composite choice named {choice_text!r} on variable {var.item!r}"
1614    )
1615
1616
1617def find_composite_current_choice(
1618    var: DecoderVariable,
1619    variables_by_item: dict[str, DecoderVariable],
1620    cv_bytes: dict[str, int],
1621) -> str | None:
1622    """Find the composite choice whose settings match the current state.
1623
1624    Composite variables don't have a wire representation of their own —
1625    they're syntactic sugar for a group of sub-variable writes. So the
1626    "current choice" is whichever named preset's ``<compositeSetting>``
1627    values ALL match the corresponding sub-variables' current human
1628    values. If no choice matches (the sub-variables are in an
1629    intermediate state), returns ``None`` and the UI shows a blank /
1630    "custom" selection.
1631
1632    * ``variables_by_item`` — the decoder's full variable table keyed
1633      by item name. Composite settings reference sub-variables by
1634      item; this map lets us resolve them.
1635    * ``cv_bytes`` — string-keyed CV byte snapshot (same shape as
1636      :func:`variable_extract_from_store` consumes).
1637    """
1638    if var.kind is not VariableKind.COMPOSITE:
1639        return None
1640    for cc in var.composite_choices:
1641        matched = True
1642        for setting in cc.settings:
1643            target = variables_by_item.get(setting.label)
1644            if target is None:
1645                matched = False
1646                break
1647            try:
1648                current = variable_extract_from_store(target, cv_bytes)
1649            except UnsupportedVariableKind:
1650                matched = False
1651                break
1652            if current != setting.value:
1653                matched = False
1654                break
1655        if matched and cc.settings:
1656            return cc.choice
1657    return None