"""The ``ComputeBackend`` contract: six methods every execution target implements.
Interface only -- no subprocess, no SSH, nothing executable lives here. The
concrete backends plug in behind it: ``local`` (subprocess, Phase 1.4) and
``ssh_exec`` (SSH / SLURM, Phase 4), looked up by ``kind`` through
``registry.py``. Keeping the interface free of any execution detail is the
"separate what from where" thesis applied to the compute plane: a stage builds a
portable ``JobSpec`` and never learns which machine ran it; making the screening
stage run on a remote GPU box is a config change, not a code change.
Adding a backend is one new subclass plus one ``register(kind, cls)`` call --
no stage, orchestrator, or CLI change. If adding a backend seems to require
touching a stage, the abstraction is leaking: stop.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
from matmelt.core.jobspec import (
Capabilities,
Handle,
HealthStatus,
JobResult,
JobSpec,
JobStatus,
)
[docs]
class ComputeBackend(ABC):
"""One uniform interface over every execution target.
Lifecycle contract -- the order a driver calls these in::
capabilities() / health_check() # before any work is handed over
handle = submit(spec) # non-blocking: work starts, handle returned
while not poll(handle).is_terminal:
...
result = fetch(handle) # faithful exit code + streams + outputs
release(handle) # idempotent cleanup, always (finally)
Invariants every implementation must hold (do not weaken):
* **Two failure tiers.** A job that runs but exits non-zero is a normal
``JobResult`` with a non-zero ``exit_code`` -- NOT an exception. Only
*infrastructure* failures (unreachable target, unknown handle, malformed
spec) raise ``BackendError``. Judging scientific success is the stage's
job; the backend only reports faithfully what happened.
* **Handles are durable.** A ``Handle`` must keep resolving after the tool
restarts (native id + persisted paths, never a live object), so an
interrupted ``matmelt run`` re-polls its in-flight jobs instead of
orphaning or double-submitting them.
* **``health_check`` never raises.** Being down is the normal answer it
reports.
* **No hardcoded compute details.** Hosts, paths, queues, credentials come
from the ``options`` dict a backend is constructed with (config /
``env:NAME`` references) -- never a string literal in backend code.
"""
def __init__(self, name: str, options: dict[str, Any]) -> None:
"""The construction contract the registry relies on: ``cls(name, options)``.
Subclasses call ``super().__init__(name, options)`` and then validate
the options they need.
Parameters
----------
name
The user's configured name for this backend instance. Two backends
can share a ``kind``; names are unique, and this is what goes into
``Handle.backend_name``.
options
The backend's free-form config block. It is copied, so later
caller-side mutation cannot reach the backend.
Raises
------
BackendError
From a subclass, for a missing or invalid option.
"""
self.name = name
self.options: dict[str, Any] = dict(options)
[docs]
@abstractmethod
def capabilities(self) -> Capabilities:
"""Report what this backend offers (cpu/gpu/slurm, ``max_parallel``).
Pure introspection -- nothing is submitted, nothing mutates. A stage's
``Resources`` needs are asserted against this before any work is handed
over, so an impossible placement (a GPU stage on a CPU-only backend) is
caught up front rather than as a late job failure.
Returns
-------
Capabilities
What this target offers. ``max_parallel`` is advertised here and
enforced by the orchestrator, not the backend.
"""
[docs]
@abstractmethod
def submit(self, spec: JobSpec) -> Handle:
"""Start the work described by ``spec``; return a durable ``Handle`` immediately.
Non-blocking: stages ``spec.input_artifacts`` as the target requires (in
place locally, uploaded remotely), launches the command, and returns --
progress is observed through ``poll``, never by waiting here.
Parameters
----------
spec
The portable job description built by a stage.
Returns
-------
Handle
Durable identity for the started job, safe to persist and re-resolve
after a restart.
Raises
------
BackendError
Only when the backend cannot *accept* the job: unreachable target,
malformed spec, an input it cannot stage. A job that starts and then
fails is reported through ``poll`` / ``fetch``, never raised here.
"""
[docs]
@abstractmethod
def poll(self, handle: Handle) -> JobStatus:
"""Return the current ``JobStatus`` of the job behind ``handle``.
Cheap and idempotent: called repeatedly until terminal, and calling it
again after that keeps returning the same terminal status. Must work
from the handle's persisted identity alone (no in-memory job map
required), so a restarted tool can poll jobs it did not launch.
Parameters
----------
handle
The handle ``submit`` returned, possibly reloaded from the state
store's in-flight ledger in a later process.
Returns
-------
JobStatus
Where the job is now.
Raises
------
BackendError
When the handle is unknown to this backend, or the status genuinely
cannot be obtained.
"""
[docs]
@abstractmethod
def fetch(self, handle: Handle) -> JobResult:
"""Collect a finished job's result: exit code, streams, output artifacts.
Meaningful once ``poll`` reports a terminal status.
Parameters
----------
handle
The handle of a job that has reached a terminal status.
Returns
-------
JobResult
The faithful outcome. ``exit_code`` is reported, never raised:
non-zero is returned as-is, and ``None`` means "ran, code unknown"
and must never be defaulted to ``0``. ``output_artifacts`` holds the
files named in ``JobSpec.output_artifacts`` that were actually found
-- a missing one is absent from the dict, for the stage to judge.
Raises
------
BackendError
Only when the result itself cannot be retrieved: unknown handle,
lost job directory, unreachable target.
"""
[docs]
@abstractmethod
def health_check(self) -> HealthStatus:
"""Report whether this backend can take work right now. **Never raises.**
The driver checks health before handing a backend any work and simply
skips a sick backend until it recovers.
Returns
-------
HealthStatus
Being down is the normal answer: ``HealthStatus(ok=False,
detail=...)`` with a short, human-readable, **secret-free** reason
for logs and ``doctor`` output.
"""
[docs]
@abstractmethod
def release(self, handle: Handle) -> None:
"""Clean up whatever the backend holds for ``handle``. **Idempotent.**
Called by the driver of the job lifecycle in a ``finally`` block (after
``fetch``, or when abandoning a job): remove scratch dirs, kill a
still-live process, forget bookkeeping. Never judges or records the
job's result; that is the caller's job.
Parameters
----------
handle
The handle to clean up. Releasing an unknown or already-released
handle is a no-op, not an error -- restart-time cleanup must always
be safe.
"""