daggerml.api

View source
   1from __future__ import annotations
   2
   3import logging
   4import time
   5import traceback
   6from collections.abc import Iterator, Mapping, Sequence
   7from contextlib import contextmanager
   8from contextvars import ContextVar
   9from dataclasses import dataclass, field
  10from importlib import metadata
  11from tempfile import TemporaryDirectory
  12from threading import RLock
  13from typing import Any, Optional, Protocol, Union, cast, get_args, overload
  14
  15from daggerml._core import CancellationError, Dml, DmlRepoError, Error, Ref, Runnable, Uri
  16from daggerml.util import BackoffWithJitter, current_time_millis
  17
  18logger = logging.getLogger(__name__)
  19
  20Scalar = Union[str, int, float, bool, type(None), Uri, Runnable]
  21Collection = Union[list, dict]
  22ProjectionStep = Union[str, int, list[int]]
  23
  24_NO_DEFAULT_DML = object()
  25_SCOPED_DEFAULT_DML: ContextVar[object] = ContextVar("daggerml_scoped_default_dml", default=_NO_DEFAULT_DML)
  26_PROCESS_DEFAULT_DML: Optional["Dml"] = None
  27
  28
  29def _resolve_default_dml(*, create: bool = True) -> tuple["Dml", str]:
  30    scoped = _SCOPED_DEFAULT_DML.get()
  31    if scoped is not _NO_DEFAULT_DML:
  32        return cast("Dml", scoped), "scoped"
  33
  34    global _PROCESS_DEFAULT_DML
  35    if _PROCESS_DEFAULT_DML is not None:
  36        return _PROCESS_DEFAULT_DML, "process"
  37
  38    if not create:
  39        raise DmlRepoError("No default Dml is configured")
  40
  41    _PROCESS_DEFAULT_DML = Dml()
  42    return _PROCESS_DEFAULT_DML, "implicit"
  43
  44
  45def get_default_dml() -> "Dml":
  46    """Return the active default Dml runtime."""
  47    dml, _source = _resolve_default_dml(create=True)
  48    return dml
  49
  50
  51def set_default_dml(dml: "Dml") -> None:
  52    """Set the process-default Dml runtime."""
  53    global _PROCESS_DEFAULT_DML
  54    _PROCESS_DEFAULT_DML = dml
  55
  56
  57def clear_default_dml() -> None:
  58    """Clear the process-default Dml runtime."""
  59    global _PROCESS_DEFAULT_DML
  60    _PROCESS_DEFAULT_DML = None
  61
  62
  63@contextmanager
  64def use_default_dml(dml: "Dml"):
  65    """Temporarily override the default Dml runtime for the active context."""
  66    token = _SCOPED_DEFAULT_DML.set(dml)
  67    try:
  68        yield dml
  69    finally:
  70        _SCOPED_DEFAULT_DML.reset(token)
  71
  72
  73def new(
  74    name="",
  75    message="",
  76    cache_key: str | None = None,
  77    execution_id: str | None = None,
  78    tags: list[str] | None = None,
  79    dml: Dml | None = None,
  80) -> "Dag":
  81    """Create a new DAG using the active or provided Dml runtime."""
  82    runtime = dml or get_default_dml()
  83    execution = Ref(f"index:{execution_id}") if execution_id is not None else None
  84    create_kwargs = {"cache_key": cache_key, "execution": execution}
  85    if tags is not None:
  86        create_kwargs["tags"] = tags
  87    index_id = runtime.runtime.create(**create_kwargs)
  88    return Dag(dml=runtime, token=index_id, name=name, message=message)
  89
  90
  91def load(
  92    name: str,
  93    dml: Dml | None = None,
  94    *,
  95    revision: Ref | str = "HEAD",
  96    remote: bool = False,
  97    dep: str | None = None,
  98) -> "Dag":
  99    """Load a DAG using the active default Dml runtime."""
 100    dml = dml or get_default_dml()
 101    dag_ref = dml.show(revision, remote=remote, dep=dep)["dags"].get(name)
 102    if dag_ref is None:
 103        raise DmlRepoError(f"DAG not found: {name}")
 104    return Dag(dml=dml, ref=dag_ref, name=name)
 105
 106
 107def resume(
 108    frozen: Ref,
 109    *,
 110    name: str,
 111    message: str,
 112    dml: Dml | None = None,
 113) -> "Dag":
 114    """Resume a frozen DAG runtime with explicitly supplied commit metadata."""
 115    runtime = dml or get_default_dml()
 116    index_id = runtime.runtime.unfreeze(frozen)
 117    return Dag(dml=runtime, token=index_id, name=name, message=message)
 118
 119
 120@contextmanager
 121def temporary(prefix="dml-tmp-", **kw):
 122    """Create a temporary Dml runtime with an unborn attached HEAD."""
 123    with TemporaryDirectory(prefix=prefix) as tmpdir:
 124        yield Dml.init(project_home=tmpdir, **kw)
 125
 126
 127def status() -> dict[str, object]:
 128    """Return status for the active default Dml runtime."""
 129    dml, source = _resolve_default_dml(create=True)
 130    return {
 131        "default": {
 132            "source": source,
 133            "has_scoped_override": _SCOPED_DEFAULT_DML.get() is not _NO_DEFAULT_DML,
 134            "has_process_default": _PROCESS_DEFAULT_DML is not None,
 135        },
 136        "status": dml.status(),
 137    }
 138
 139
 140def _make_node(dag: "Dag", ref: Ref) -> "Node":
 141    """
 142    Create a Node from a Dag and Ref.
 143
 144    Parameters
 145    ----------
 146    dag : Dag
 147        The parent DAG.
 148    ref : Ref
 149        The reference to the node.
 150    Returns
 151    -------
 152    Node
 153        A Node instance representing the reference in the DAG.
 154    """
 155    node_value = dag.dml.dag.get_node(ref)
 156    if isinstance(node_value, Error):
 157        raise NodeError(node_value, node_ref=ref, dag=dag)
 158    info: dict[str, Any] = {"data_type": type(node_value).__name__.lower()}
 159    # Determine node type based on value and populate info
 160    if isinstance(node_value, list):
 161        info["length"] = len(node_value)
 162        node = ListNode(dag, ref, _info=info)
 163    elif isinstance(node_value, dict):
 164        info["length"] = len(node_value)
 165        info["keys"] = sorted(node_value.keys())
 166        node = DictNode(dag, ref, _info=info)
 167    elif isinstance(node_value, Runnable):
 168        node = RunnableNode(dag, ref, _info=info)
 169    else:
 170        node = ScalarNode(dag, ref, _info=info)
 171    return node
 172
 173
 174def _info_for_value(value: Any) -> dict[str, Any]:
 175    info: dict[str, Any] = {"data_type": type(value).__name__.lower()}
 176    if isinstance(value, list):
 177        info["length"] = len(value)
 178    elif isinstance(value, dict):
 179        info["length"] = len(value)
 180        info["keys"] = sorted(value.keys())
 181    return info
 182
 183
 184def _normalize_projection_step(key: ProjectionStep | slice, *, length: int | None = None) -> ProjectionStep:
 185    if isinstance(key, slice):
 186        if key.step is not None:
 187            raise ValueError("Slice step is not supported")
 188        start = key.start if key.start is not None else 0
 189        if key.stop is None:
 190            if length is None:
 191                raise DmlRepoError("Slice stop requires known collection length")
 192            stop = length
 193        else:
 194            stop = key.stop
 195        return [start, stop]
 196    return key
 197
 198
 199def _apply_projection_step(value: Any, step: ProjectionStep) -> Any:
 200    if isinstance(step, list):
 201        if len(step) != 2:
 202            raise DmlRepoError("Slice projection requires exactly [start, stop]")
 203        return value[slice(*step)]
 204    return value[step]
 205
 206
 207def _apply_projection_path(value: Any, path: tuple[ProjectionStep, ...]) -> Any:
 208    for step in path:
 209        value = _apply_projection_step(value, step)
 210    return value
 211
 212
 213def _describe_node(node: "Node") -> Mapping[str, Any]:
 214    return node.dag.dml.dag.describe_node(node.ref)
 215
 216
 217def _builtin_name_for_argv(dag: "Dag", argv_refs: list[Ref]) -> str | None:
 218    if not argv_refs:
 219        return None
 220    runnable = dag.dml.dag.get_node(argv_refs[0], recursive=True)
 221    if not isinstance(runnable, Runnable):
 222        return None
 223    if runnable.adapter != "":
 224        return None
 225    if not runnable.target.uri.startswith("daggerml:"):
 226        return None
 227    return runnable.target.uri.split(":", 1)[1]
 228
 229
 230def _prepend_get_path_step(path: tuple[ProjectionStep, ...], key: ProjectionStep) -> tuple[ProjectionStep, ...] | None:
 231    if isinstance(key, list):
 232        if len(key) != 2:
 233            return None
 234        if not path:
 235            return None
 236        first, *rest = path
 237        if not isinstance(first, int):
 238            return None
 239        return (key[0] + first, *rest)
 240    return (key, *path)
 241
 242
 243def _backtrack_builtin(
 244    node: "Node", argv_refs: list[Ref], path: tuple[ProjectionStep, ...]
 245) -> tuple["Node", tuple[ProjectionStep, ...]] | None:
 246    builtin = _builtin_name_for_argv(node.dag, argv_refs)
 247    if builtin is None:
 248        return None
 249    arg_nodes = [_make_node(node.dag, ref) for ref in argv_refs[1:]]
 250    if builtin == "get":
 251        if len(arg_nodes) < 2:
 252            return None
 253        key = cast(ProjectionStep, arg_nodes[1].value())
 254        next_path = _prepend_get_path_step(path, key)
 255        if next_path is None:
 256            return None
 257        return arg_nodes[0], next_path
 258    if builtin == "list":
 259        if not path:
 260            return None
 261        index, *rest = path
 262        if not isinstance(index, int):
 263            return None
 264        if index < 0:
 265            index += len(arg_nodes)
 266        if index < 0 or index >= len(arg_nodes):
 267            return None
 268        return arg_nodes[index], tuple(rest)
 269    if builtin == "dict":
 270        if not path:
 271            return None
 272        key, *rest = path
 273        if not isinstance(key, str):
 274            return None
 275        for idx in range(0, len(arg_nodes), 2):
 276            if idx + 1 >= len(arg_nodes):
 277                break
 278            if arg_nodes[idx].value() == key:
 279                return arg_nodes[idx + 1], tuple(rest)
 280        return None
 281    if builtin == "assoc":
 282        if len(arg_nodes) < 3 or not path:
 283            return None
 284        selected_key, *rest = path
 285        assoc_key = arg_nodes[1].value()
 286        if selected_key == assoc_key:
 287            return arg_nodes[2], tuple(rest)
 288        return arg_nodes[0], path
 289    if builtin == "conj":
 290        if len(arg_nodes) < 2 or not path:
 291            return None
 292        index, *rest = path
 293        if not isinstance(index, int):
 294            return None
 295        base_len = len(arg_nodes[0].value())
 296        if index < 0:
 297            index += base_len + 1
 298        if index == base_len:
 299            return arg_nodes[1], tuple(rest)
 300        return arg_nodes[0], path
 301    return None
 302
 303
 304def _nearest_context_state(
 305    node: "Node", path: tuple[ProjectionStep, ...]
 306) -> tuple[Dag, Node, tuple[ProjectionStep, ...], bool]:
 307    current = node
 308    current_path = path
 309    while True:
 310        node_info = _describe_node(current)
 311        node_type = node_info["type"]
 312        if node_type == "ImportNode":
 313            source_dag = Dag(dml=current.dag.dml, ref=cast(Ref, node_info["dag"]))
 314            source_node = _make_node(source_dag, cast(Ref, node_info["node"]))
 315            if current_path:
 316                current = source_node
 317                continue
 318            return source_dag, source_node, current_path, True
 319        if node_type == "FnNode":
 320            argv_refs = cast(list[Ref], node_info["argv"])
 321            source = _backtrack_builtin(current, argv_refs, current_path)
 322            if source is not None:
 323                current, current_path = source
 324                continue
 325            fn_dag = Dag(dml=current.dag.dml, ref=cast(Ref, node_info["dag"]))
 326            return fn_dag, fn_dag.result, current_path, True
 327        return current.dag, current, current_path, False
 328
 329
 330def _resolve_context(node: "Node", path: tuple[ProjectionStep, ...], *, root: bool) -> Dag:
 331    context_dag, next_node, next_path, can_recurse = _nearest_context_state(node, path)
 332    if not root:
 333        return context_dag
 334    if not can_recurse:
 335        return context_dag
 336    return _resolve_context(next_node, next_path, root=True)
 337
 338
 339@dataclass
 340class Dag:
 341    dml: Dml
 342    token: Optional[Ref] = None  # Working index id
 343    ref: Optional[Ref] = None
 344    name: str = ""  # DAG name for commit
 345    message: str = ""  # Commit message
 346
 347    def __repr__(self):
 348        to = self.ref.to if self.ref else (self.token.to if self.token is not None else "NA")
 349        return f"Dag({to})"
 350
 351    def __hash__(self):
 352        "Useful only for tests."
 353        return 42
 354
 355    def __eq__(self, other):
 356        "DAG equality is based on identity, not content."
 357        if not isinstance(other, Dag):
 358            return False
 359        return self.ref == other.ref and self.token == other.token and self.dml == other.dml
 360
 361    def __enter__(self):
 362        "Catch exceptions and commit an Error"
 363        assert not self.ref
 364        return self
 365
 366    def __exit__(self, exc_type, exc_value, tb):
 367        if exc_value is not None:
 368            # Convert exception to Error and commit it
 369            traceback.print_exception(exc_type, exc_value, tb)
 370            err = Error.from_ex(exc_value) if not isinstance(exc_value, Error) else exc_value
 371            self.commit(err)
 372
 373    def _require_index_ref(self) -> Ref:
 374        if self.token is None:
 375            raise DmlRepoError("No active index")
 376        return self.token
 377
 378    def _read_dag_ref(self) -> Ref:
 379        """Return the completed or partial DAG backing this wrapper's reads."""
 380        if self.ref is not None:
 381            return self.ref
 382        return cast(Ref, self.dml.runtime.describe(self._require_index_ref())["dag"])
 383
 384    @property
 385    def tags(self) -> list[str]:
 386        """Return the normalized tags stored on this DAG."""
 387        return list(self.dml.dag.describe(self._read_dag_ref())["tags"])
 388
 389    def _put_literal(self, value: Any, *, name: Optional[str] = None) -> Ref:
 390        index_id = self._require_index_ref()
 391        value = apply_codecs(value, dag=self)
 392        return self.dml.runtime.put_literal(index_id, value, name=name)
 393
 394    def _start_fn(self, argv: list[Ref], *, name: Optional[str] = None) -> Optional[Ref]:
 395        return self.dml.runtime.start_fn(self._require_index_ref(), argv, name=name)
 396
 397    def _call_builtin(self, uri: str, *args: Any, name: Optional[str] = None) -> Ref:
 398        fn_ref = self._put_literal(Runnable(target=Uri(uri), kwargs={}, adapter=""))
 399        argv: list[Ref] = [fn_ref]
 400        for arg in args:
 401            argv.append(arg if isinstance(arg, Ref) else self._put_literal(arg))
 402        result = self._start_fn(argv, name=name)
 403        if result is None:
 404            raise DmlRepoError("Function execution failed")
 405        return result
 406
 407    def __len__(self) -> int:
 408        return len(self.keys())
 409
 410    def __iter__(self):
 411        yield from self.keys()
 412
 413    def _get_named_node(self, name: str) -> "Node":
 414        node_ref = self.dml.dag.describe(self._read_dag_ref())["names"].get(name)
 415        if node_ref is None:
 416            raise DmlRepoError(f"Node '{name}' not found in DAG")
 417        return _make_node(self, node_ref)
 418
 419    def _set_named_node(self, name: str, value: Any) -> None:
 420        if self.ref is not None:
 421            raise DmlRepoError("Cannot set node names on a committed DAG.")
 422        if isinstance(value, Node):
 423            value = value.ref
 424        if isinstance(value, Ref):
 425            self.dml.runtime.set_node_name(self._require_index_ref(), name, value)
 426            return
 427        self.put(value, name=name)
 428
 429    def __getitem__(self, name: str) -> "Node":
 430        return self._get_named_node(name)
 431
 432    def __setitem__(self, name: str, value: Any) -> None:
 433        self.put(value, name=name)
 434
 435    def __getattr__(self, name: str) -> "Node":
 436        if name.startswith("_"):
 437            raise AttributeError(name)
 438        return self[name]
 439
 440    def __setattr__(self, name: str, value: Any) -> None:
 441        dataclass_fields = getattr(type(self), "__dataclass_fields__", {})
 442        if name in dataclass_fields or hasattr(type(self), name):
 443            object.__setattr__(self, name, value)
 444            return
 445        self[name] = value
 446
 447    def keys(self) -> list[str]:
 448        """Get the list of all node names in the dag"""
 449        names_dict = self.dml.dag.describe(self._read_dag_ref())["names"]
 450        return sorted(names_dict.keys())
 451
 452    def values(self) -> list["Node"]:
 453        """Get the list of all nodes in the dag"""
 454        names_dict = self.dml.dag.describe(self._read_dag_ref())["names"]
 455        return [_make_node(self, ref) for ref in names_dict.values()]
 456
 457    @property
 458    def argv(self) -> "ListNode":
 459        "Access the dag's argv node"
 460        argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"]
 461        if not isinstance(argv_ref, Ref):
 462            raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv")
 463        return cast(ListNode, _make_node(self, argv_ref))
 464
 465    @property
 466    def result(self) -> "Node":
 467        """Get the result node of the dag"""
 468        if self.ref is None:
 469            raise DmlRepoError("Cannot access result of an uncommitted DAG")
 470        description = self.dml.dag.describe(self.ref)
 471        error_ref = description.get("error")
 472        if isinstance(error_ref, Ref):
 473            raise self.dml.dag.get_error(error_ref)
 474        ref = description.get("result")
 475        if not isinstance(ref, Ref):
 476            raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet")
 477        return _make_node(self, ref)
 478
 479    @overload
 480    def put(self, value: Union[list, "ListNode"], *, name=None) -> "ListNode": ...
 481    @overload
 482    def put(self, value: Union[dict, "DictNode"], *, name=None) -> "DictNode": ...
 483    @overload
 484    def put(self, value: Union[Runnable, "RunnableNode"], *, name=None) -> "RunnableNode": ...
 485    @overload
 486    def put(self, value: Union[Scalar, "ScalarNode"], *, name=None) -> "ScalarNode": ...
 487    @overload
 488    def put(self, value: Any, *, name=None) -> "Node": ...
 489    def put(self, value: Any, *, name=None) -> "Node":
 490        """
 491        Add a value to the DAG.
 492
 493        Parameters
 494        ----------
 495        value : Union[Scalar, Collection]
 496            Value to add
 497        name : str, optional
 498            Name for the node
 499        Returns
 500        -------
 501        Node
 502            Node representing the value
 503
 504        Examples
 505        --------
 506        >>> n1 = dag.put(42, name="answer")
 507        >>> n1.value()
 508        42
 509        >>> n2 = dag.put({"a": 1, "b": [n1, "23"]})
 510        >>> n2.value()
 511        {'a': 1, 'b': [42, '23']}
 512        >>> n3 = dag.put({"a": 1, "b": [n1, "23"]})
 513        >>> n3.value()
 514        {'a': 1, 'b': [42, '23']}
 515        """
 516        return _make_node(self, self._put_literal(value, name=name))
 517
 518    def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node":
 519        """
 520        Import a node from a different (committed) DAG into the current DAG.
 521
 522        Parameters
 523        ----------
 524        dag_name : str
 525            Name of the DAG to import from
 526        node_name : str, optional
 527            Name of the node to import. If None, imports the result node of the DAG.
 528
 529        Returns
 530        -------
 531        Node
 532            The loaded node or DAG
 533
 534        Examples
 535        --------
 536        >>> dag = new(dml=dml, name="test", message="test")
 537        >>> n1 = dag.put(42, name="answer")
 538        >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data")
 539        >>> dag.commit(n2)
 540        >>> dag2 = new(dml=dml, name="test2", message="test2")
 541        >>> imported_n2 = dag2.require("test", "data", name="imported_data")
 542        >>> imported_n2.value()
 543        {'a': 1, 'b': [42, '23']}
 544        """
 545        index = self._require_index_ref()
 546        if isinstance(dag_name, Dag):
 547            if dag_name.ref is None:
 548                raise DmlRepoError("Cannot import an uncommitted DAG")
 549            dag_ref = dag_name.ref
 550        else:
 551            commit = self.dml.runtime.describe(index)["parents"][0]
 552            dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name)
 553            if dag_ref is None:
 554                raise DmlRepoError(f"DAG not found: {dag_name}")
 555        dag_info = self.dml.dag.describe(dag_ref)
 556        node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result")
 557        if node_ref is None:
 558            raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'")
 559        node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name)
 560        return _make_node(self, node_ref)
 561
 562    def call(
 563        self,
 564        fn: Any,
 565        *args: Any,
 566        name: Optional[str] = None,
 567        sleep: Optional[callable] = None,
 568        timeout: int = -1,
 569    ) -> "Node":
 570        """
 571        Call a function node with arguments.
 572
 573        Parameters
 574        ----------
 575        fn : Union[Runnable, RunnableNode]
 576            Function to call
 577        *args : Union[Node, Scalar, Collection]
 578            Arguments to pass to the function
 579        name : str, optional
 580            Name for the result node
 581        sleep : callable, optional
 582            A nullary function that returns sleep time in milliseconds
 583        timeout : int, default=-1
 584            Maximum time to wait in milliseconds. If <= 0, wait indefinitely.
 585
 586        Returns
 587        -------
 588        Node
 589            Result node
 590
 591        Raises
 592        ------
 593        TimeoutError
 594            If the function call exceeds the timeout
 595        Error
 596            If the function returns an error
 597        """
 598        sleep = sleep or BackoffWithJitter()
 599        argv_seed = [fn, *args]
 600        end = current_time_millis() + timeout
 601        while timeout <= 0 or current_time_millis() < end:
 602            argv_refs = [self._put_literal(value) for value in argv_seed]
 603            resp = self._start_fn(argv_refs, name=name)
 604            if resp:
 605                return _make_node(self, resp)
 606            time.sleep(sleep() / 1000)
 607        raise TimeoutError(f"invoking function: {fn}")
 608
 609    def commit(self, value) -> None:
 610        """
 611        Commit a value to the DAG.
 612
 613        Parameters
 614        ----------
 615        value : Union[Node, Error, Any]
 616            Value to commit
 617        """
 618        # errors are committed as-is, everything else is a node
 619        if not isinstance(value, (Error, Node)):
 620            value = self.put(value)
 621        if isinstance(value, Node):
 622            value = value.ref
 623        self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name)
 624        self.token = None  # Clear the working index since it's now committed
 625
 626    def freeze(self, message: str | None = None) -> "Dag":
 627        """Freeze this uncommitted DAG's runtime index for read-only inspection."""
 628        freeze_message = f"dag: {self.name}"
 629        if message:
 630            freeze_message = f"{freeze_message}\n{message}"
 631        self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message)
 632        return self
 633
 634    def unfreeze(self) -> "Dag":
 635        """Unfreeze this uncommitted DAG's runtime index."""
 636        self.token = self.dml.runtime.unfreeze(self._require_index_ref())
 637        return self
 638
 639    def dump(self) -> dict[str, Any]:
 640        """Return this active or frozen runtime's repository objects as JSON-compatible data.
 641
 642        This does not freeze execution or preserve locks, execution records, or
 643        external artifact bytes. ``dml.runtime.load(data)`` restores the original
 644        index ID and overwrites an existing copy.
 645        """
 646        return self.dml.runtime.dump(self._require_index_ref())
 647
 648    @classmethod
 649    def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag":
 650        """Construct a DAG wrapper from a runtimepack, preserving its index ID.
 651
 652        Parameters
 653        ----------
 654        data : dict
 655            Runtimepack returned by ``Dag.dump()``.
 656        dml : Dml, optional
 657            Destination repository. Defaults to the active default repository.
 658
 659        Returns
 660        -------
 661        Dag
 662            The loaded active or frozen runtime. An existing index is overwritten;
 663            execution records and locks are neither created nor restored.
 664        """
 665        runtime = dml if dml is not None else get_default_dml()
 666        return cls(dml=runtime, token=runtime.runtime.load(data))
 667
 668    def cancel(self, max_retries: int = 3):
 669        """Cancel the DAG's execution.
 670
 671        Parameters
 672        ----------
 673        max_retries : int, default=3
 674            Maximum cancellation retries after the initial attempt.
 675        """
 676        if self.token is None:
 677            raise DmlRepoError("Cannot cancel a committed DAG")
 678        logger.info(f"Cancelling execution {self.token}")
 679        self.dml.runtime.cancel(self.token, max_retries=max_retries)
 680        self.token = None  # Clear the index ref to indicate it's no longer active
 681        raise CancellationError("DAG execution cancelled")
 682
 683
 684class NodeError(Error):
 685    """A stored error enriched with the node that produced it."""
 686
 687    def __init__(self, error: Error, *, node_ref: Ref, dag: Dag):
 688        super().__init__(message=error.message, origin=error.origin, type=error.type, stack=list(error.stack))
 689        self.node_ref = node_ref
 690        self.dag = dag
 691
 692    def context(self) -> Dag:
 693        """Return the function DAG that recorded this node's failure."""
 694        dag = self.dag
 695        node_ref = self.node_ref
 696        while True:
 697            info = dag.dml.dag.describe_node(node_ref)
 698            if info["type"] == "FnNode":
 699                return Dag(dml=dag.dml, ref=info["dag"])
 700            if info["type"] != "ImportNode":
 701                raise DmlRepoError(f"Node {node_ref} does not resolve to a failed function call")
 702            dag = Dag(dml=dag.dml, ref=info["dag"])
 703            node_ref = info["node"]
 704
 705
 706@dataclass(frozen=True)
 707class Node:  # noqa: F811
 708    """
 709    Representation of a node in a DaggerML DAG.
 710
 711    Parameters
 712    ----------
 713    dag : Dag
 714        Parent DAG
 715    ref : Ref
 716        Node reference
 717    """
 718
 719    dag: Dag
 720    ref: Ref
 721    _info: dict = field(default_factory=dict)
 722
 723    def __repr__(self):
 724        ref_id = self.ref if isinstance(self.ref, Error) else self.ref.to
 725        return f"{self.__class__.__name__}({ref_id})"
 726
 727    def __hash__(self):
 728        return hash(self.ref)
 729
 730    def __eq__(self, other):
 731        if not isinstance(other, Node):
 732            return NotImplemented
 733        return self.ref == other.ref
 734
 735    def context(self, *, root: bool = True) -> Dag:
 736        """
 737        Resolve the provenance context (DAG) for this node.
 738
 739        This follows import/function provenance while treating builtin
 740        collection-construction and selection DAGs as transparent.
 741
 742        Parameters
 743        ----------
 744        root : bool, default=True
 745            If False, return the nearest sub-DAG in which this value exists as a
 746            proper node across a non-builtin import/function boundary. If True,
 747            continue recursively until provenance no longer crosses a
 748            non-builtin import/function boundary and return that first rooted
 749            context.
 750
 751        Returns
 752        -------
 753        Dag
 754            The nearest or rooted provenance DAG for this node.
 755
 756        Examples
 757        --------
 758        >>> source = new(dml=dml, name="source", message="source")
 759        >>> answer = source.put(42, name="answer")
 760        >>> payload = source.put({"answer": answer}, name="payload")
 761        >>> source.commit(payload)
 762        >>> consumer = new(dml=dml, name="consumer", message="consumer")
 763        >>> imported = consumer.require("source", "payload", name="payload")
 764        >>> consumer.commit(imported)
 765        >>> loaded = load("consumer", dml=dml)
 766        >>> loaded.result["answer"].context() == source
 767        True
 768        """
 769        return _resolve_context(self, (), root=root)
 770
 771    @property
 772    def type(self):
 773        """Get the data type of the node."""
 774        return self._info["data_type"]
 775
 776    @overload
 777    def value(self: "ScalarNode") -> Scalar: ...
 778    @overload
 779    def value(self: "ListNode") -> list: ...
 780    @overload
 781    def value(self: "DictNode") -> dict: ...
 782    @overload
 783    def value(self: "RunnableNode") -> Runnable: ...
 784    @overload
 785    def value(self: "Node") -> Any: ...
 786    def value(self):
 787        """
 788        Get the concrete value of this node.
 789
 790        Returns
 791        -------
 792        Any
 793            The actual value represented by this node
 794        """
 795        value = self.dag.dml.dag.get_node(self.ref, recursive=True)
 796        if isinstance(value, Error):
 797            raise NodeError(value, node_ref=self.ref, dag=self.dag)
 798        return value
 799
 800    def __call__(self, *args, name=None, sleep=None, timeout=-1, **kw) -> "Node":
 801        raise TypeError(f"Node of type '{self.type}' is not callable")
 802
 803
 804class ScalarNode(Node):
 805    pass
 806
 807
 808class RunnableNode(Node):
 809    def __call__(self, *args, name=None, sleep=None, timeout=-1) -> "Node":
 810        """
 811        Call this node as a function.
 812
 813        Parameters
 814        ----------
 815        *args : Any
 816            Arguments to pass to the function
 817        name : str, optional
 818            Name for the result node
 819        sleep : callable, optional
 820            A nullary function that returns sleep time in milliseconds
 821        timeout : int, default=-1
 822            Maximum time to wait in milliseconds. -1 means wait forever.
 823
 824        Returns
 825        -------
 826        Node
 827            Result node
 828
 829        Raises
 830        ------
 831        TimeoutError
 832            If the function call exceeds the timeout
 833        Error
 834            If the function returns an error
 835        """
 836        return self.dag.call(self, *args, name=name, sleep=sleep, timeout=timeout)
 837
 838
 839@dataclass(frozen=True)
 840class Projection:
 841    dag: Dag
 842    base: Node
 843    path: tuple[ProjectionStep, ...]
 844    _info: dict = field(default_factory=dict)
 845
 846    def __repr__(self):
 847        return f"Projection({self.base!r}, path={self.path!r})"
 848
 849    @classmethod
 850    def from_step(cls, base: Node, step: ProjectionStep) -> "Projection":
 851        return cls(dag=base.dag, base=base, path=(step,))
 852
 853    def _extend(self, step: ProjectionStep) -> "Projection":
 854        return Projection(dag=self.dag, base=self.base, path=(*self.path, step))
 855
 856    def value(self):
 857        return _apply_projection_path(self.base.value(), self.path)
 858
 859    def context(self, *, root: bool = True) -> Dag:
 860        return _resolve_context(self.base, self.path, root=root)
 861
 862    def __call__(self, *args, **kwargs):
 863        raise TypeError(f"Projection of type '{self.type}' is not callable")
 864
 865    @property
 866    def type(self):
 867        if "data_type" in self._info:
 868            return self._info["data_type"]
 869        return _info_for_value(self.value())["data_type"]
 870
 871    def __len__(self):
 872        if "length" in self._info:
 873            return self._info["length"]
 874        value = self.value()
 875        if not isinstance(value, (list, dict)):
 876            raise TypeError(f"Object of type '{type(value).__name__}' has no len()")
 877        return len(value)
 878
 879    def __iter__(self):
 880        value = self.value()
 881        if isinstance(value, list):
 882            for i in range(len(value)):
 883                yield self[i]
 884            return
 885        if isinstance(value, dict):
 886            yield from self.keys()
 887            return
 888        raise TypeError(f"Object of type '{type(value).__name__}' is not iterable")
 889
 890    def __getitem__(self, key: ProjectionStep | slice) -> "Projection":
 891        value = self.value()
 892        if isinstance(value, dict):
 893            if not isinstance(key, str):
 894                raise TypeError(f"Dict keys must be strings but got {type(key).__name__}")
 895            step = cast(ProjectionStep, key)
 896        elif isinstance(value, list):
 897            if not isinstance(key, (int, slice)):
 898                raise TypeError(f"List indices must be integers or slices but got {type(key).__name__}")
 899            step = _normalize_projection_step(cast(ProjectionStep | slice, key), length=len(value))
 900        else:
 901            raise TypeError(f"Cannot project into object of type '{type(value).__name__}'")
 902        projected = self._extend(step)
 903        return Projection(dag=self.dag, base=self.base, path=projected.path, _info=_info_for_value(projected.value()))
 904
 905    def keys(self) -> list[str]:
 906        value = self.value()
 907        if not isinstance(value, dict):
 908            raise TypeError(f"Cannot get keys of type: {type(value).__name__}")
 909        return sorted(value.keys())
 910
 911
 912class CollectionNode(Node):  # noqa: F811
 913    """
 914    Representation of a collection node in a DaggerML DAG.
 915
 916    Parameters
 917    ----------
 918    dag : Dag
 919        Parent DAG
 920    ref : Ref
 921        Node reference
 922    """
 923
 924    def contains(self, item, *, name=None) -> "ScalarNode":
 925        """
 926        For collection nodes, checks to see if `item` is in `self`
 927
 928        Returns
 929        -------
 930        Node
 931            Node with the boolean of is `item` in `self`
 932        """
 933        item_ref = item.ref if isinstance(item, Node) else item
 934        result = self.dag._call_builtin("daggerml:contains", self.ref, item_ref, name=name)
 935        return cast(ScalarNode, _make_node(self.dag, result))
 936
 937    def __contains__(self, item):
 938        return self.contains(item).value()  # has to return boolean
 939
 940    def __len__(self):  # python requires this to be an int
 941        """
 942        Get the node's length
 943
 944        Returns
 945        -------
 946        Node
 947            Node with the length of the collection
 948
 949        Raises
 950        ------
 951        Error
 952            If the node isn't a collection (e.g. list or dict).
 953        """
 954        return self._info["length"]
 955
 956
 957class ListNode(CollectionNode):  # noqa: F811
 958    """
 959    Representation of a collection node in a DaggerML DAG.
 960
 961    Parameters
 962    ----------
 963    dag : Dag
 964        Parent DAG
 965    ref : Ref
 966        Node reference
 967    """
 968
 969    @overload
 970    def __getitem__(self, key: Union[slice, list[int]]) -> Union["ListNode", Projection]: ...
 971    @overload
 972    def __getitem__(self, key: Union[int, "Node"]) -> Union["Node", Projection]: ...
 973    def __getitem__(self, key: Union[slice, list[int], int, "Node"]) -> Union["Node", Projection]:
 974        if self.dag.ref is not None:
 975            if isinstance(key, Node):
 976                raise TypeError("Committed list projections require concrete int or slice keys")
 977            step = _normalize_projection_step(key, length=len(self))
 978            return Projection(
 979                dag=self.dag,
 980                base=self,
 981                path=(step,),
 982                _info=_info_for_value(_apply_projection_step(self.value(), step)),
 983            )
 984        if isinstance(key, slice):
 985            if key.step is not None:
 986                raise ValueError("Slice step is not supported")
 987            start = key.start if key.start is not None else 0
 988            stop = key.stop if key.stop is not None else len(self)
 989            key = [start, stop]
 990        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key))
 991
 992    def __iter__(self):
 993        """
 994        Iterate over the node's values (items if it's a list, and keys if it's a
 995        dict)
 996
 997        Returns
 998        -------
 999        Node
1000            Result node
1001
1002        Raises
1003        ------
1004        Error
1005            If the node isn't a collection (e.g. list or dict).
1006        """
1007        for i in range(len(self)):
1008            yield self[i]
1009
1010    def conj(self, item, *, name=None) -> "ListNode":
1011        """
1012        For a list node, append an item
1013
1014        Returns
1015        -------
1016        Node
1017            Node containing the new collection
1018
1019        Notes
1020        -----
1021        `append` is an alias `conj`
1022        """
1023        item_ref = item.ref if isinstance(item, Node) else item
1024        resp = self.dag._call_builtin("daggerml:conj", self.ref, item_ref, name=name)
1025        return cast(ListNode, _make_node(self.dag, resp))
1026
1027    def append(self, item, *, name=None) -> "ListNode":
1028        """
1029        For a list node, append an item
1030
1031        Returns
1032        -------
1033        Node
1034            Node containing the new collection
1035
1036        See Also
1037        --------
1038        conj : The main implementation
1039        """
1040        return self.conj(item, name=name)
1041
1042
1043class DictNode(CollectionNode):  # noqa: F811
1044    def __getitem__(self, key: Union[str, "Node"]) -> Union["Node", Projection]:
1045        if self.dag.ref is not None:
1046            if not isinstance(key, str):
1047                raise TypeError(f"Dict keys must be strings but got {type(key).__name__}")
1048            return Projection(dag=self.dag, base=self, path=(key,), _info=_info_for_value(self.value()[key]))
1049        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key))
1050
1051    def keys(self) -> list[str]:
1052        """
1053        Get the keys of a dictionary node.
1054
1055        Parameters
1056        ----------
1057        name : str, optional
1058            Name for the result node
1059
1060        Returns
1061        -------
1062        list[str]
1063            List of keys in the dictionary node
1064        """
1065        return self._info["keys"].copy()
1066
1067    def __iter__(self):
1068        """
1069        Iterate over the node's values (items if it's a list, and keys if it's a
1070        dict)
1071
1072        Returns
1073        -------
1074        Node
1075            Result node
1076
1077        Raises
1078        ------
1079        Error
1080            If the node isn't a collection (e.g. list or dict).
1081        """
1082        for k in self.keys():
1083            yield k
1084
1085    def get(self, key, default=None, *, name=None) -> "Node":
1086        """
1087        For a dict node, return the value for key if key exists, else default.
1088
1089        If default is not given, it defaults to None, so that this method never raises a KeyError.
1090        """
1091        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key, default, name=name))
1092
1093    def items(self) -> Iterator[tuple[str, "Node|Projection"]]:
1094        """
1095        Iterate over key-value pairs of a dictionary node.
1096
1097        Returns
1098        -------
1099        Iterator[tuple[Node, Node]]
1100            Iterator over (key, value) pairs
1101        """
1102        if self.type != "dict":
1103            raise Error(f"Cannot iterate items of type: {self.type}", origin="dml", type="TypeError")
1104        for k in self:
1105            yield k, self[k]
1106
1107    def values(self) -> list["Node|Projection"]:
1108        """
1109        Get the values of a dictionary node.
1110
1111        Parameters
1112        ----------
1113        name : str, optional
1114            Name for the result node
1115
1116        Returns
1117        -------
1118        list[Node]
1119            List of values in the dictionary node
1120        """
1121        return [self[k] for k in self]
1122
1123    def assoc(self, key, value, *, name=None) -> "DictNode":
1124        """
1125        For a dict node, associate a new value into the map
1126
1127        Returns
1128        -------
1129        Node
1130            Node containing the new dict
1131        """
1132        value_ref = value.ref if isinstance(value, Node) else value
1133        resp = self.dag._call_builtin("daggerml:assoc", self.ref, key, value_ref, name=name)
1134        return cast(DictNode, _make_node(self.dag, resp))
1135
1136    def update(self, update) -> "DictNode":
1137        """
1138        For a dict node, update like python dicts
1139
1140        Returns
1141        -------
1142        Node
1143            Node containing the new collection
1144
1145        Notes
1146        -----
1147        calls `assoc` iteratively for k, v pairs in update.
1148
1149        See Also
1150        --------
1151        assoc : The main implementation
1152        """
1153        for k, v in update.items():
1154            self = self.assoc(k, v)
1155        return self
1156
1157
1158################################################################################
1159################## Codec system for encoding literals in DAGs ##################
1160################################################################################
1161
1162LITERAL_CODEC_ENTRYPOINT_GROUP = "daggerml.codecs"
1163_codecs: list[tuple[int, int, "LiteralCodec"]] = []
1164_plugins_loaded = False
1165_lock = RLock()
1166
1167
1168class LiteralCodec(Protocol):
1169    def can_encode(self, value: Any) -> bool: ...
1170
1171    def encode(self, value: Any, dag: "Dag") -> Any: ...
1172
1173
1174class CodecError(Error):
1175    def __init__(self, message: str):
1176        super().__init__(message, origin="dml-codec", type="codec-error")
1177
1178
1179def _entry_points(group=LITERAL_CODEC_ENTRYPOINT_GROUP) -> list[metadata.EntryPoint]:
1180    points = metadata.entry_points()
1181    result = list(points.select(group=group))
1182    result.sort(key=lambda ep: (ep.name, ep.value))
1183    return result
1184
1185
1186def ensure_literal_codec_plugins_loaded() -> None:
1187    global _plugins_loaded
1188    if _plugins_loaded:
1189        return
1190    with _lock:
1191        codec_seq = 0
1192        if _plugins_loaded:
1193            return
1194        loaded = []
1195        for entry_point in _entry_points():
1196            try:
1197                registrations = entry_point.load()()
1198                for item in registrations:
1199                    priority, codec = item
1200                    codec_seq += 1
1201                    loaded.append((priority, codec_seq, codec))
1202            except Exception as e:
1203                msg = f"Literal codec plugin '{entry_point.name} ({entry_point.value})' failed: {e}"
1204                raise CodecError(msg) from None
1205        loaded.sort(key=lambda item: (-item[0], item[1]))
1206        _codecs.extend(loaded)
1207        _plugins_loaded = True
1208
1209
1210def iter_codecs() -> Iterator[LiteralCodec]:
1211    ensure_literal_codec_plugins_loaded()
1212    yield from [codec for _priority, _seq, codec in _codecs]
1213
1214
1215def apply_codec(value: Any, *, dag: Dag) -> Any:
1216    for codec in iter_codecs():
1217        try:
1218            if codec.can_encode(value):
1219                resp = codec.encode(value, dag)
1220                if isinstance(resp, type(value)):
1221                    codec_name = codec.__class__.__name__
1222                    msg = f"Literal codec {codec_name} encoded {value.__class__.__name__} to {resp.__class__.__name__}."
1223                    raise CodecError(msg)
1224                return resp
1225        except Exception as e:
1226            if isinstance(e, DmlRepoError):
1227                raise
1228            raise CodecError(f"Literal codec {codec.__class__.__name__} failed: {e}") from e
1229    raise CodecError(f"No codec found for value of type {type(value).__name__}")
1230
1231
1232def apply_codecs(value: Any, *, dag: Dag) -> Any:
1233    while not isinstance(value, (*get_args(Scalar), *get_args(Collection), Ref)):
1234        value = apply_codec(value, dag=dag)
1235    if isinstance(value, list):
1236        return [apply_codecs(v, dag=dag) for v in value]
1237    if isinstance(value, dict):
1238        return {k: apply_codecs(v, dag=dag) for k, v in value.items()}
1239    if isinstance(value, Uri):
1240        return Uri(apply_codecs(value.uri, dag=dag))
1241    if isinstance(value, Runnable):
1242        target = apply_codecs(value.target, dag=dag)
1243        sub = apply_codecs(value.sub, dag=dag)
1244        kwargs = {k: apply_codecs(v, dag=dag) for k, v in value.kwargs.items()}
1245        return Runnable(target=target, adapter=value.adapter, kwargs=kwargs, sub=sub)
1246    return value
1247
1248
1249class MiscPyTypeCodec:
1250    def can_encode(self, value: Any) -> bool:
1251        return isinstance(value, Mapping) or (
1252            isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray))
1253        )
1254
1255    def encode(self, value: Sequence | Mapping, dag: Dag) -> Any:
1256        if isinstance(value, Mapping):
1257            return {k: apply_codecs(v, dag=dag) for k, v in value.items()}
1258        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
1259            return [apply_codecs(v, dag=dag) for v in value]
1260
1261
1262class NodeCodec:
1263    def can_encode(self, value: Any) -> bool:
1264        return isinstance(value, Node)
1265
1266    def encode(self, value: "Node", dag: Dag) -> Ref:
1267        assert dag.token is not None, "DAG must have a token to encode nodes"
1268        if value.dag.token is not None and value.dag.token == dag.token:
1269            return value.ref
1270        if value.dag.ref is None:
1271            raise CodecError("Cannot encode node from uncommitted DAG in a different index")
1272        try:
1273            return dag.dml.runtime.put_import(dag._require_index_ref(), value.dag.ref, node=value.ref, name=None)
1274        except Exception as e:
1275            raise CodecError(f"Failed to encode cross-dag node import: {e}") from e
1276
1277
1278class ProjectionCodec:
1279    def can_encode(self, value: Any) -> bool:
1280        return isinstance(value, Projection)
1281
1282    def encode(self, value: "Projection", dag: Dag) -> Ref:
1283        assert dag.token is not None, "DAG must have a token to encode projections"
1284        node_ref = NodeCodec().encode(value.base, dag)
1285        for step in value.path:
1286            node_ref = dag._call_builtin("daggerml:get", node_ref, step)
1287        return node_ref
1288
1289
1290def codecs() -> list:
1291    return [(0, NodeCodec()), (0, MiscPyTypeCodec()), (0, ProjectionCodec())]

logger

logger= <Logger daggerml.api (WARNING)>

Scalar

Scalar= typing.Union[str, int, float, bool, NoneType, daggerml.Uri, daggerml.Runnable]

Collection

Collection= typing.Union[list, dict]

ProjectionStep

ProjectionStep= typing.Union[str, int, list[int]]

get_default_dml

def get_default_dml() -> daggerml.Dml:
View source
46def get_default_dml() -> "Dml":
47    """Return the active default Dml runtime."""
48    dml, _source = _resolve_default_dml(create=True)
49    return dml

Return the active default Dml runtime.

set_default_dml

def set_default_dml(dml: daggerml.Dml) -> None:
View source
52def set_default_dml(dml: "Dml") -> None:
53    """Set the process-default Dml runtime."""
54    global _PROCESS_DEFAULT_DML
55    _PROCESS_DEFAULT_DML = dml

Set the process-default Dml runtime.

clear_default_dml

def clear_default_dml() -> None:
View source
58def clear_default_dml() -> None:
59    """Clear the process-default Dml runtime."""
60    global _PROCESS_DEFAULT_DML
61    _PROCESS_DEFAULT_DML = None

Clear the process-default Dml runtime.

use_default_dml

@contextmanager
def use_default_dml(dml: daggerml.Dml):
View source
64@contextmanager
65def use_default_dml(dml: "Dml"):
66    """Temporarily override the default Dml runtime for the active context."""
67    token = _SCOPED_DEFAULT_DML.set(dml)
68    try:
69        yield dml
70    finally:
71        _SCOPED_DEFAULT_DML.reset(token)

Temporarily override the default Dml runtime for the active context.

new

def new( name='', message='', cache_key: str | None = None, execution_id: str | None = None, tags: list[str] | None = None, dml: daggerml.Dml | None = None) -> Dag:
View source
74def new(
75    name="",
76    message="",
77    cache_key: str | None = None,
78    execution_id: str | None = None,
79    tags: list[str] | None = None,
80    dml: Dml | None = None,
81) -> "Dag":
82    """Create a new DAG using the active or provided Dml runtime."""
83    runtime = dml or get_default_dml()
84    execution = Ref(f"index:{execution_id}") if execution_id is not None else None
85    create_kwargs = {"cache_key": cache_key, "execution": execution}
86    if tags is not None:
87        create_kwargs["tags"] = tags
88    index_id = runtime.runtime.create(**create_kwargs)
89    return Dag(dml=runtime, token=index_id, name=name, message=message)

Create a new DAG using the active or provided Dml runtime.

load

def load( name: str, dml: daggerml.Dml | None = None, *, revision: daggerml.Ref | str = 'HEAD', remote: bool = False, dep: str | None = None) -> Dag:
View source
 92def load(
 93    name: str,
 94    dml: Dml | None = None,
 95    *,
 96    revision: Ref | str = "HEAD",
 97    remote: bool = False,
 98    dep: str | None = None,
 99) -> "Dag":
100    """Load a DAG using the active default Dml runtime."""
101    dml = dml or get_default_dml()
102    dag_ref = dml.show(revision, remote=remote, dep=dep)["dags"].get(name)
103    if dag_ref is None:
104        raise DmlRepoError(f"DAG not found: {name}")
105    return Dag(dml=dml, ref=dag_ref, name=name)

Load a DAG using the active default Dml runtime.

resume

def resume( frozen: daggerml.Ref, *, name: str, message: str, dml: daggerml.Dml | None = None) -> Dag:
View source
108def resume(
109    frozen: Ref,
110    *,
111    name: str,
112    message: str,
113    dml: Dml | None = None,
114) -> "Dag":
115    """Resume a frozen DAG runtime with explicitly supplied commit metadata."""
116    runtime = dml or get_default_dml()
117    index_id = runtime.runtime.unfreeze(frozen)
118    return Dag(dml=runtime, token=index_id, name=name, message=message)

Resume a frozen DAG runtime with explicitly supplied commit metadata.

temporary

@contextmanager
def temporary(prefix='dml-tmp-', **kw):
View source
121@contextmanager
122def temporary(prefix="dml-tmp-", **kw):
123    """Create a temporary Dml runtime with an unborn attached HEAD."""
124    with TemporaryDirectory(prefix=prefix) as tmpdir:
125        yield Dml.init(project_home=tmpdir, **kw)

Create a temporary Dml runtime with an unborn attached HEAD.

status

def status() -> dict[str, object]:
View source
128def status() -> dict[str, object]:
129    """Return status for the active default Dml runtime."""
130    dml, source = _resolve_default_dml(create=True)
131    return {
132        "default": {
133            "source": source,
134            "has_scoped_override": _SCOPED_DEFAULT_DML.get() is not _NO_DEFAULT_DML,
135            "has_process_default": _PROCESS_DEFAULT_DML is not None,
136        },
137        "status": dml.status(),
138    }

Return status for the active default Dml runtime.

Dag

@dataclass
class Dag:
View source
340@dataclass
341class Dag:
342    dml: Dml
343    token: Optional[Ref] = None  # Working index id
344    ref: Optional[Ref] = None
345    name: str = ""  # DAG name for commit
346    message: str = ""  # Commit message
347
348    def __repr__(self):
349        to = self.ref.to if self.ref else (self.token.to if self.token is not None else "NA")
350        return f"Dag({to})"
351
352    def __hash__(self):
353        "Useful only for tests."
354        return 42
355
356    def __eq__(self, other):
357        "DAG equality is based on identity, not content."
358        if not isinstance(other, Dag):
359            return False
360        return self.ref == other.ref and self.token == other.token and self.dml == other.dml
361
362    def __enter__(self):
363        "Catch exceptions and commit an Error"
364        assert not self.ref
365        return self
366
367    def __exit__(self, exc_type, exc_value, tb):
368        if exc_value is not None:
369            # Convert exception to Error and commit it
370            traceback.print_exception(exc_type, exc_value, tb)
371            err = Error.from_ex(exc_value) if not isinstance(exc_value, Error) else exc_value
372            self.commit(err)
373
374    def _require_index_ref(self) -> Ref:
375        if self.token is None:
376            raise DmlRepoError("No active index")
377        return self.token
378
379    def _read_dag_ref(self) -> Ref:
380        """Return the completed or partial DAG backing this wrapper's reads."""
381        if self.ref is not None:
382            return self.ref
383        return cast(Ref, self.dml.runtime.describe(self._require_index_ref())["dag"])
384
385    @property
386    def tags(self) -> list[str]:
387        """Return the normalized tags stored on this DAG."""
388        return list(self.dml.dag.describe(self._read_dag_ref())["tags"])
389
390    def _put_literal(self, value: Any, *, name: Optional[str] = None) -> Ref:
391        index_id = self._require_index_ref()
392        value = apply_codecs(value, dag=self)
393        return self.dml.runtime.put_literal(index_id, value, name=name)
394
395    def _start_fn(self, argv: list[Ref], *, name: Optional[str] = None) -> Optional[Ref]:
396        return self.dml.runtime.start_fn(self._require_index_ref(), argv, name=name)
397
398    def _call_builtin(self, uri: str, *args: Any, name: Optional[str] = None) -> Ref:
399        fn_ref = self._put_literal(Runnable(target=Uri(uri), kwargs={}, adapter=""))
400        argv: list[Ref] = [fn_ref]
401        for arg in args:
402            argv.append(arg if isinstance(arg, Ref) else self._put_literal(arg))
403        result = self._start_fn(argv, name=name)
404        if result is None:
405            raise DmlRepoError("Function execution failed")
406        return result
407
408    def __len__(self) -> int:
409        return len(self.keys())
410
411    def __iter__(self):
412        yield from self.keys()
413
414    def _get_named_node(self, name: str) -> "Node":
415        node_ref = self.dml.dag.describe(self._read_dag_ref())["names"].get(name)
416        if node_ref is None:
417            raise DmlRepoError(f"Node '{name}' not found in DAG")
418        return _make_node(self, node_ref)
419
420    def _set_named_node(self, name: str, value: Any) -> None:
421        if self.ref is not None:
422            raise DmlRepoError("Cannot set node names on a committed DAG.")
423        if isinstance(value, Node):
424            value = value.ref
425        if isinstance(value, Ref):
426            self.dml.runtime.set_node_name(self._require_index_ref(), name, value)
427            return
428        self.put(value, name=name)
429
430    def __getitem__(self, name: str) -> "Node":
431        return self._get_named_node(name)
432
433    def __setitem__(self, name: str, value: Any) -> None:
434        self.put(value, name=name)
435
436    def __getattr__(self, name: str) -> "Node":
437        if name.startswith("_"):
438            raise AttributeError(name)
439        return self[name]
440
441    def __setattr__(self, name: str, value: Any) -> None:
442        dataclass_fields = getattr(type(self), "__dataclass_fields__", {})
443        if name in dataclass_fields or hasattr(type(self), name):
444            object.__setattr__(self, name, value)
445            return
446        self[name] = value
447
448    def keys(self) -> list[str]:
449        """Get the list of all node names in the dag"""
450        names_dict = self.dml.dag.describe(self._read_dag_ref())["names"]
451        return sorted(names_dict.keys())
452
453    def values(self) -> list["Node"]:
454        """Get the list of all nodes in the dag"""
455        names_dict = self.dml.dag.describe(self._read_dag_ref())["names"]
456        return [_make_node(self, ref) for ref in names_dict.values()]
457
458    @property
459    def argv(self) -> "ListNode":
460        "Access the dag's argv node"
461        argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"]
462        if not isinstance(argv_ref, Ref):
463            raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv")
464        return cast(ListNode, _make_node(self, argv_ref))
465
466    @property
467    def result(self) -> "Node":
468        """Get the result node of the dag"""
469        if self.ref is None:
470            raise DmlRepoError("Cannot access result of an uncommitted DAG")
471        description = self.dml.dag.describe(self.ref)
472        error_ref = description.get("error")
473        if isinstance(error_ref, Ref):
474            raise self.dml.dag.get_error(error_ref)
475        ref = description.get("result")
476        if not isinstance(ref, Ref):
477            raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet")
478        return _make_node(self, ref)
479
480    @overload
481    def put(self, value: Union[list, "ListNode"], *, name=None) -> "ListNode": ...
482    @overload
483    def put(self, value: Union[dict, "DictNode"], *, name=None) -> "DictNode": ...
484    @overload
485    def put(self, value: Union[Runnable, "RunnableNode"], *, name=None) -> "RunnableNode": ...
486    @overload
487    def put(self, value: Union[Scalar, "ScalarNode"], *, name=None) -> "ScalarNode": ...
488    @overload
489    def put(self, value: Any, *, name=None) -> "Node": ...
490    def put(self, value: Any, *, name=None) -> "Node":
491        """
492        Add a value to the DAG.
493
494        Parameters
495        ----------
496        value : Union[Scalar, Collection]
497            Value to add
498        name : str, optional
499            Name for the node
500        Returns
501        -------
502        Node
503            Node representing the value
504
505        Examples
506        --------
507        >>> n1 = dag.put(42, name="answer")
508        >>> n1.value()
509        42
510        >>> n2 = dag.put({"a": 1, "b": [n1, "23"]})
511        >>> n2.value()
512        {'a': 1, 'b': [42, '23']}
513        >>> n3 = dag.put({"a": 1, "b": [n1, "23"]})
514        >>> n3.value()
515        {'a': 1, 'b': [42, '23']}
516        """
517        return _make_node(self, self._put_literal(value, name=name))
518
519    def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node":
520        """
521        Import a node from a different (committed) DAG into the current DAG.
522
523        Parameters
524        ----------
525        dag_name : str
526            Name of the DAG to import from
527        node_name : str, optional
528            Name of the node to import. If None, imports the result node of the DAG.
529
530        Returns
531        -------
532        Node
533            The loaded node or DAG
534
535        Examples
536        --------
537        >>> dag = new(dml=dml, name="test", message="test")
538        >>> n1 = dag.put(42, name="answer")
539        >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data")
540        >>> dag.commit(n2)
541        >>> dag2 = new(dml=dml, name="test2", message="test2")
542        >>> imported_n2 = dag2.require("test", "data", name="imported_data")
543        >>> imported_n2.value()
544        {'a': 1, 'b': [42, '23']}
545        """
546        index = self._require_index_ref()
547        if isinstance(dag_name, Dag):
548            if dag_name.ref is None:
549                raise DmlRepoError("Cannot import an uncommitted DAG")
550            dag_ref = dag_name.ref
551        else:
552            commit = self.dml.runtime.describe(index)["parents"][0]
553            dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name)
554            if dag_ref is None:
555                raise DmlRepoError(f"DAG not found: {dag_name}")
556        dag_info = self.dml.dag.describe(dag_ref)
557        node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result")
558        if node_ref is None:
559            raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'")
560        node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name)
561        return _make_node(self, node_ref)
562
563    def call(
564        self,
565        fn: Any,
566        *args: Any,
567        name: Optional[str] = None,
568        sleep: Optional[callable] = None,
569        timeout: int = -1,
570    ) -> "Node":
571        """
572        Call a function node with arguments.
573
574        Parameters
575        ----------
576        fn : Union[Runnable, RunnableNode]
577            Function to call
578        *args : Union[Node, Scalar, Collection]
579            Arguments to pass to the function
580        name : str, optional
581            Name for the result node
582        sleep : callable, optional
583            A nullary function that returns sleep time in milliseconds
584        timeout : int, default=-1
585            Maximum time to wait in milliseconds. If <= 0, wait indefinitely.
586
587        Returns
588        -------
589        Node
590            Result node
591
592        Raises
593        ------
594        TimeoutError
595            If the function call exceeds the timeout
596        Error
597            If the function returns an error
598        """
599        sleep = sleep or BackoffWithJitter()
600        argv_seed = [fn, *args]
601        end = current_time_millis() + timeout
602        while timeout <= 0 or current_time_millis() < end:
603            argv_refs = [self._put_literal(value) for value in argv_seed]
604            resp = self._start_fn(argv_refs, name=name)
605            if resp:
606                return _make_node(self, resp)
607            time.sleep(sleep() / 1000)
608        raise TimeoutError(f"invoking function: {fn}")
609
610    def commit(self, value) -> None:
611        """
612        Commit a value to the DAG.
613
614        Parameters
615        ----------
616        value : Union[Node, Error, Any]
617            Value to commit
618        """
619        # errors are committed as-is, everything else is a node
620        if not isinstance(value, (Error, Node)):
621            value = self.put(value)
622        if isinstance(value, Node):
623            value = value.ref
624        self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name)
625        self.token = None  # Clear the working index since it's now committed
626
627    def freeze(self, message: str | None = None) -> "Dag":
628        """Freeze this uncommitted DAG's runtime index for read-only inspection."""
629        freeze_message = f"dag: {self.name}"
630        if message:
631            freeze_message = f"{freeze_message}\n{message}"
632        self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message)
633        return self
634
635    def unfreeze(self) -> "Dag":
636        """Unfreeze this uncommitted DAG's runtime index."""
637        self.token = self.dml.runtime.unfreeze(self._require_index_ref())
638        return self
639
640    def dump(self) -> dict[str, Any]:
641        """Return this active or frozen runtime's repository objects as JSON-compatible data.
642
643        This does not freeze execution or preserve locks, execution records, or
644        external artifact bytes. ``dml.runtime.load(data)`` restores the original
645        index ID and overwrites an existing copy.
646        """
647        return self.dml.runtime.dump(self._require_index_ref())
648
649    @classmethod
650    def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag":
651        """Construct a DAG wrapper from a runtimepack, preserving its index ID.
652
653        Parameters
654        ----------
655        data : dict
656            Runtimepack returned by ``Dag.dump()``.
657        dml : Dml, optional
658            Destination repository. Defaults to the active default repository.
659
660        Returns
661        -------
662        Dag
663            The loaded active or frozen runtime. An existing index is overwritten;
664            execution records and locks are neither created nor restored.
665        """
666        runtime = dml if dml is not None else get_default_dml()
667        return cls(dml=runtime, token=runtime.runtime.load(data))
668
669    def cancel(self, max_retries: int = 3):
670        """Cancel the DAG's execution.
671
672        Parameters
673        ----------
674        max_retries : int, default=3
675            Maximum cancellation retries after the initial attempt.
676        """
677        if self.token is None:
678            raise DmlRepoError("Cannot cancel a committed DAG")
679        logger.info(f"Cancelling execution {self.token}")
680        self.dml.runtime.cancel(self.token, max_retries=max_retries)
681        self.token = None  # Clear the index ref to indicate it's no longer active
682        raise CancellationError("DAG execution cancelled")

Dag.__init__

Dag( dml: daggerml.Dml, token: Optional[daggerml.Ref] = None, ref: Optional[daggerml.Ref] = None, name: str = '', message: str = '')

Dag.dml

Dag.token

token: Optional[daggerml.Ref]= None

Dag.ref

ref: Optional[daggerml.Ref]= None

Dag.name

name: str= ''

Dag.message

message: str= ''

Dag.tags

tags: list[str]
View source
385    @property
386    def tags(self) -> list[str]:
387        """Return the normalized tags stored on this DAG."""
388        return list(self.dml.dag.describe(self._read_dag_ref())["tags"])

Return the normalized tags stored on this DAG.

Dag.keys

def keys(self) -> list[str]:
View source
448    def keys(self) -> list[str]:
449        """Get the list of all node names in the dag"""
450        names_dict = self.dml.dag.describe(self._read_dag_ref())["names"]
451        return sorted(names_dict.keys())

Get the list of all node names in the dag

Dag.values

def values(self) -> list[Node]:
View source
453    def values(self) -> list["Node"]:
454        """Get the list of all nodes in the dag"""
455        names_dict = self.dml.dag.describe(self._read_dag_ref())["names"]
456        return [_make_node(self, ref) for ref in names_dict.values()]

Get the list of all nodes in the dag

Dag.argv

argv: ListNode
View source
458    @property
459    def argv(self) -> "ListNode":
460        "Access the dag's argv node"
461        argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"]
462        if not isinstance(argv_ref, Ref):
463            raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv")
464        return cast(ListNode, _make_node(self, argv_ref))

Access the dag's argv node

Dag.result

result: Node
View source
466    @property
467    def result(self) -> "Node":
468        """Get the result node of the dag"""
469        if self.ref is None:
470            raise DmlRepoError("Cannot access result of an uncommitted DAG")
471        description = self.dml.dag.describe(self.ref)
472        error_ref = description.get("error")
473        if isinstance(error_ref, Ref):
474            raise self.dml.dag.get_error(error_ref)
475        ref = description.get("result")
476        if not isinstance(ref, Ref):
477            raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet")
478        return _make_node(self, ref)

Get the result node of the dag

Dag.put

def put(self, value: Any, *, name=None) -> Node:
View source
490    def put(self, value: Any, *, name=None) -> "Node":
491        """
492        Add a value to the DAG.
493
494        Parameters
495        ----------
496        value : Union[Scalar, Collection]
497            Value to add
498        name : str, optional
499            Name for the node
500        Returns
501        -------
502        Node
503            Node representing the value
504
505        Examples
506        --------
507        >>> n1 = dag.put(42, name="answer")
508        >>> n1.value()
509        42
510        >>> n2 = dag.put({"a": 1, "b": [n1, "23"]})
511        >>> n2.value()
512        {'a': 1, 'b': [42, '23']}
513        >>> n3 = dag.put({"a": 1, "b": [n1, "23"]})
514        >>> n3.value()
515        {'a': 1, 'b': [42, '23']}
516        """
517        return _make_node(self, self._put_literal(value, name=name))

Add a value to the DAG.

Parameters
  • value (Union[Scalar, Collection]): Value to add
  • name (str, optional): Name for the node
Returns
  • Node: Node representing the value
Examples
>>> n1 = dag.put(42, name="answer")
>>> n1.value()
42
>>> n2 = dag.put({"a": 1, "b": [n1, "23"]})
>>> n2.value()
{'a': 1, 'b': [42, '23']}
>>> n3 = dag.put({"a": 1, "b": [n1, "23"]})
>>> n3.value()
{'a': 1, 'b': [42, '23']}

Dag.require

def require( self, dag_name: "str | 'Dag'", node_name: str | None = None, *, name: str | None = None) -> Node:
View source
519    def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node":
520        """
521        Import a node from a different (committed) DAG into the current DAG.
522
523        Parameters
524        ----------
525        dag_name : str
526            Name of the DAG to import from
527        node_name : str, optional
528            Name of the node to import. If None, imports the result node of the DAG.
529
530        Returns
531        -------
532        Node
533            The loaded node or DAG
534
535        Examples
536        --------
537        >>> dag = new(dml=dml, name="test", message="test")
538        >>> n1 = dag.put(42, name="answer")
539        >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data")
540        >>> dag.commit(n2)
541        >>> dag2 = new(dml=dml, name="test2", message="test2")
542        >>> imported_n2 = dag2.require("test", "data", name="imported_data")
543        >>> imported_n2.value()
544        {'a': 1, 'b': [42, '23']}
545        """
546        index = self._require_index_ref()
547        if isinstance(dag_name, Dag):
548            if dag_name.ref is None:
549                raise DmlRepoError("Cannot import an uncommitted DAG")
550            dag_ref = dag_name.ref
551        else:
552            commit = self.dml.runtime.describe(index)["parents"][0]
553            dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name)
554            if dag_ref is None:
555                raise DmlRepoError(f"DAG not found: {dag_name}")
556        dag_info = self.dml.dag.describe(dag_ref)
557        node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result")
558        if node_ref is None:
559            raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'")
560        node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name)
561        return _make_node(self, node_ref)

Import a node from a different (committed) DAG into the current DAG.

Parameters
  • dag_name (str): Name of the DAG to import from
  • node_name (str, optional): Name of the node to import. If None, imports the result node of the DAG.
Returns
  • Node: The loaded node or DAG
Examples
>>> dag = new(dml=dml, name="test", message="test")
>>> n1 = dag.put(42, name="answer")
>>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data")
>>> dag.commit(n2)
>>> dag2 = new(dml=dml, name="test2", message="test2")
>>> imported_n2 = dag2.require("test", "data", name="imported_data")
>>> imported_n2.value()
{'a': 1, 'b': [42, '23']}

Dag.call

def call( self, fn: Any, *args: Any, name: Optional[str] = None, sleep: Optional[<built-in function callable>] = None, timeout: int = -1) -> Node:
View source
563    def call(
564        self,
565        fn: Any,
566        *args: Any,
567        name: Optional[str] = None,
568        sleep: Optional[callable] = None,
569        timeout: int = -1,
570    ) -> "Node":
571        """
572        Call a function node with arguments.
573
574        Parameters
575        ----------
576        fn : Union[Runnable, RunnableNode]
577            Function to call
578        *args : Union[Node, Scalar, Collection]
579            Arguments to pass to the function
580        name : str, optional
581            Name for the result node
582        sleep : callable, optional
583            A nullary function that returns sleep time in milliseconds
584        timeout : int, default=-1
585            Maximum time to wait in milliseconds. If <= 0, wait indefinitely.
586
587        Returns
588        -------
589        Node
590            Result node
591
592        Raises
593        ------
594        TimeoutError
595            If the function call exceeds the timeout
596        Error
597            If the function returns an error
598        """
599        sleep = sleep or BackoffWithJitter()
600        argv_seed = [fn, *args]
601        end = current_time_millis() + timeout
602        while timeout <= 0 or current_time_millis() < end:
603            argv_refs = [self._put_literal(value) for value in argv_seed]
604            resp = self._start_fn(argv_refs, name=name)
605            if resp:
606                return _make_node(self, resp)
607            time.sleep(sleep() / 1000)
608        raise TimeoutError(f"invoking function: {fn}")

Call a function node with arguments.

Parameters
  • fn (Union[Runnable, RunnableNode]): Function to call
  • *args (Union[Node, Scalar, Collection]): Arguments to pass to the function
  • name (str, optional): Name for the result node
  • sleep (callable, optional): A nullary function that returns sleep time in milliseconds
  • timeout (int, default=-1): Maximum time to wait in milliseconds. If <= 0, wait indefinitely.
Returns
  • Node: Result node
Raises
  • TimeoutError: If the function call exceeds the timeout
  • Error: If the function returns an error

Dag.commit

def commit(self, value) -> None:
View source
610    def commit(self, value) -> None:
611        """
612        Commit a value to the DAG.
613
614        Parameters
615        ----------
616        value : Union[Node, Error, Any]
617            Value to commit
618        """
619        # errors are committed as-is, everything else is a node
620        if not isinstance(value, (Error, Node)):
621            value = self.put(value)
622        if isinstance(value, Node):
623            value = value.ref
624        self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name)
625        self.token = None  # Clear the working index since it's now committed

Commit a value to the DAG.

Parameters
  • value (Union[Node, Error, Any]): Value to commit

Dag.freeze

def freeze(self, message: str | None = None) -> Dag:
View source
627    def freeze(self, message: str | None = None) -> "Dag":
628        """Freeze this uncommitted DAG's runtime index for read-only inspection."""
629        freeze_message = f"dag: {self.name}"
630        if message:
631            freeze_message = f"{freeze_message}\n{message}"
632        self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message)
633        return self

Freeze this uncommitted DAG's runtime index for read-only inspection.

Dag.unfreeze

def unfreeze(self) -> Dag:
View source
635    def unfreeze(self) -> "Dag":
636        """Unfreeze this uncommitted DAG's runtime index."""
637        self.token = self.dml.runtime.unfreeze(self._require_index_ref())
638        return self

Unfreeze this uncommitted DAG's runtime index.

Dag.dump

def dump(self) -> dict[str, typing.Any]:
View source
640    def dump(self) -> dict[str, Any]:
641        """Return this active or frozen runtime's repository objects as JSON-compatible data.
642
643        This does not freeze execution or preserve locks, execution records, or
644        external artifact bytes. ``dml.runtime.load(data)`` restores the original
645        index ID and overwrites an existing copy.
646        """
647        return self.dml.runtime.dump(self._require_index_ref())

Return this active or frozen runtime's repository objects as JSON-compatible data.

This does not freeze execution or preserve locks, execution records, or external artifact bytes. dml.runtime.load(data) restores the original index ID and overwrites an existing copy.

Dag.load

@classmethod
def load( cls, data: dict[str, typing.Any], *, dml: daggerml.Dml | None = None) -> Dag:
View source
649    @classmethod
650    def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag":
651        """Construct a DAG wrapper from a runtimepack, preserving its index ID.
652
653        Parameters
654        ----------
655        data : dict
656            Runtimepack returned by ``Dag.dump()``.
657        dml : Dml, optional
658            Destination repository. Defaults to the active default repository.
659
660        Returns
661        -------
662        Dag
663            The loaded active or frozen runtime. An existing index is overwritten;
664            execution records and locks are neither created nor restored.
665        """
666        runtime = dml if dml is not None else get_default_dml()
667        return cls(dml=runtime, token=runtime.runtime.load(data))

Construct a DAG wrapper from a runtimepack, preserving its index ID.

Parameters
  • data (dict): Runtimepack returned by Dag.dump().
  • dml (Dml, optional): Destination repository. Defaults to the active default repository.
Returns
  • Dag: The loaded active or frozen runtime. An existing index is overwritten; execution records and locks are neither created nor restored.

Dag.cancel

def cancel(self, max_retries: int = 3):
View source
669    def cancel(self, max_retries: int = 3):
670        """Cancel the DAG's execution.
671
672        Parameters
673        ----------
674        max_retries : int, default=3
675            Maximum cancellation retries after the initial attempt.
676        """
677        if self.token is None:
678            raise DmlRepoError("Cannot cancel a committed DAG")
679        logger.info(f"Cancelling execution {self.token}")
680        self.dml.runtime.cancel(self.token, max_retries=max_retries)
681        self.token = None  # Clear the index ref to indicate it's no longer active
682        raise CancellationError("DAG execution cancelled")

Cancel the DAG's execution.

Parameters
  • max_retries (int, default=3): Maximum cancellation retries after the initial attempt.

NodeError

class NodeError(daggerml._core.types.Error):
View source
685class NodeError(Error):
686    """A stored error enriched with the node that produced it."""
687
688    def __init__(self, error: Error, *, node_ref: Ref, dag: Dag):
689        super().__init__(message=error.message, origin=error.origin, type=error.type, stack=list(error.stack))
690        self.node_ref = node_ref
691        self.dag = dag
692
693    def context(self) -> Dag:
694        """Return the function DAG that recorded this node's failure."""
695        dag = self.dag
696        node_ref = self.node_ref
697        while True:
698            info = dag.dml.dag.describe_node(node_ref)
699            if info["type"] == "FnNode":
700                return Dag(dml=dag.dml, ref=info["dag"])
701            if info["type"] != "ImportNode":
702                raise DmlRepoError(f"Node {node_ref} does not resolve to a failed function call")
703            dag = Dag(dml=dag.dml, ref=info["dag"])
704            node_ref = info["node"]

A stored error enriched with the node that produced it.

NodeError.__init__

NodeError( error: daggerml.Error, *, node_ref: daggerml.Ref, dag: Dag)
View source
688    def __init__(self, error: Error, *, node_ref: Ref, dag: Dag):
689        super().__init__(message=error.message, origin=error.origin, type=error.type, stack=list(error.stack))
690        self.node_ref = node_ref
691        self.dag = dag

NodeError.node_ref

node_ref

NodeError.dag

dag

NodeError.context

def context(self) -> Dag:
View source
693    def context(self) -> Dag:
694        """Return the function DAG that recorded this node's failure."""
695        dag = self.dag
696        node_ref = self.node_ref
697        while True:
698            info = dag.dml.dag.describe_node(node_ref)
699            if info["type"] == "FnNode":
700                return Dag(dml=dag.dml, ref=info["dag"])
701            if info["type"] != "ImportNode":
702                raise DmlRepoError(f"Node {node_ref} does not resolve to a failed function call")
703            dag = Dag(dml=dag.dml, ref=info["dag"])
704            node_ref = info["node"]

Return the function DAG that recorded this node's failure.

Node

@dataclass(frozen=True)
class Node:
View source
707@dataclass(frozen=True)
708class Node:  # noqa: F811
709    """
710    Representation of a node in a DaggerML DAG.
711
712    Parameters
713    ----------
714    dag : Dag
715        Parent DAG
716    ref : Ref
717        Node reference
718    """
719
720    dag: Dag
721    ref: Ref
722    _info: dict = field(default_factory=dict)
723
724    def __repr__(self):
725        ref_id = self.ref if isinstance(self.ref, Error) else self.ref.to
726        return f"{self.__class__.__name__}({ref_id})"
727
728    def __hash__(self):
729        return hash(self.ref)
730
731    def __eq__(self, other):
732        if not isinstance(other, Node):
733            return NotImplemented
734        return self.ref == other.ref
735
736    def context(self, *, root: bool = True) -> Dag:
737        """
738        Resolve the provenance context (DAG) for this node.
739
740        This follows import/function provenance while treating builtin
741        collection-construction and selection DAGs as transparent.
742
743        Parameters
744        ----------
745        root : bool, default=True
746            If False, return the nearest sub-DAG in which this value exists as a
747            proper node across a non-builtin import/function boundary. If True,
748            continue recursively until provenance no longer crosses a
749            non-builtin import/function boundary and return that first rooted
750            context.
751
752        Returns
753        -------
754        Dag
755            The nearest or rooted provenance DAG for this node.
756
757        Examples
758        --------
759        >>> source = new(dml=dml, name="source", message="source")
760        >>> answer = source.put(42, name="answer")
761        >>> payload = source.put({"answer": answer}, name="payload")
762        >>> source.commit(payload)
763        >>> consumer = new(dml=dml, name="consumer", message="consumer")
764        >>> imported = consumer.require("source", "payload", name="payload")
765        >>> consumer.commit(imported)
766        >>> loaded = load("consumer", dml=dml)
767        >>> loaded.result["answer"].context() == source
768        True
769        """
770        return _resolve_context(self, (), root=root)
771
772    @property
773    def type(self):
774        """Get the data type of the node."""
775        return self._info["data_type"]
776
777    @overload
778    def value(self: "ScalarNode") -> Scalar: ...
779    @overload
780    def value(self: "ListNode") -> list: ...
781    @overload
782    def value(self: "DictNode") -> dict: ...
783    @overload
784    def value(self: "RunnableNode") -> Runnable: ...
785    @overload
786    def value(self: "Node") -> Any: ...
787    def value(self):
788        """
789        Get the concrete value of this node.
790
791        Returns
792        -------
793        Any
794            The actual value represented by this node
795        """
796        value = self.dag.dml.dag.get_node(self.ref, recursive=True)
797        if isinstance(value, Error):
798            raise NodeError(value, node_ref=self.ref, dag=self.dag)
799        return value
800
801    def __call__(self, *args, name=None, sleep=None, timeout=-1, **kw) -> "Node":
802        raise TypeError(f"Node of type '{self.type}' is not callable")

Representation of a node in a DaggerML DAG.

Parameters
  • dag (Dag): Parent DAG
  • ref (Ref): Node reference

Node.__init__

Node( dag: Dag, ref: daggerml.Ref, _info: dict = <factory>)

Node.dag

dag: Dag

Node.ref

Node.context

def context(self, *, root: bool = True) -> Dag:
View source
736    def context(self, *, root: bool = True) -> Dag:
737        """
738        Resolve the provenance context (DAG) for this node.
739
740        This follows import/function provenance while treating builtin
741        collection-construction and selection DAGs as transparent.
742
743        Parameters
744        ----------
745        root : bool, default=True
746            If False, return the nearest sub-DAG in which this value exists as a
747            proper node across a non-builtin import/function boundary. If True,
748            continue recursively until provenance no longer crosses a
749            non-builtin import/function boundary and return that first rooted
750            context.
751
752        Returns
753        -------
754        Dag
755            The nearest or rooted provenance DAG for this node.
756
757        Examples
758        --------
759        >>> source = new(dml=dml, name="source", message="source")
760        >>> answer = source.put(42, name="answer")
761        >>> payload = source.put({"answer": answer}, name="payload")
762        >>> source.commit(payload)
763        >>> consumer = new(dml=dml, name="consumer", message="consumer")
764        >>> imported = consumer.require("source", "payload", name="payload")
765        >>> consumer.commit(imported)
766        >>> loaded = load("consumer", dml=dml)
767        >>> loaded.result["answer"].context() == source
768        True
769        """
770        return _resolve_context(self, (), root=root)

Resolve the provenance context (DAG) for this node.

This follows import/function provenance while treating builtin collection-construction and selection DAGs as transparent.

Parameters
  • root (bool, default=True): If False, return the nearest sub-DAG in which this value exists as a proper node across a non-builtin import/function boundary. If True, continue recursively until provenance no longer crosses a non-builtin import/function boundary and return that first rooted context.
Returns
  • Dag: The nearest or rooted provenance DAG for this node.
Examples
>>> source = new(dml=dml, name="source", message="source")
>>> answer = source.put(42, name="answer")
>>> payload = source.put({"answer": answer}, name="payload")
>>> source.commit(payload)
>>> consumer = new(dml=dml, name="consumer", message="consumer")
>>> imported = consumer.require("source", "payload", name="payload")
>>> consumer.commit(imported)
>>> loaded = load("consumer", dml=dml)
>>> loaded.result["answer"].context() == source
True

Node.type

type
View source
772    @property
773    def type(self):
774        """Get the data type of the node."""
775        return self._info["data_type"]

Get the data type of the node.

Node.value

def value(self):
View source
787    def value(self):
788        """
789        Get the concrete value of this node.
790
791        Returns
792        -------
793        Any
794            The actual value represented by this node
795        """
796        value = self.dag.dml.dag.get_node(self.ref, recursive=True)
797        if isinstance(value, Error):
798            raise NodeError(value, node_ref=self.ref, dag=self.dag)
799        return value

Get the concrete value of this node.

Returns
  • Any: The actual value represented by this node

ScalarNode

class ScalarNode(Node):
View source
805class ScalarNode(Node):
806    pass

Representation of a node in a DaggerML DAG.

Parameters
  • dag (Dag): Parent DAG
  • ref (Ref): Node reference

Inherited members

Node
Node
dag
ref
context
type
value

RunnableNode

class RunnableNode(Node):
View source
809class RunnableNode(Node):
810    def __call__(self, *args, name=None, sleep=None, timeout=-1) -> "Node":
811        """
812        Call this node as a function.
813
814        Parameters
815        ----------
816        *args : Any
817            Arguments to pass to the function
818        name : str, optional
819            Name for the result node
820        sleep : callable, optional
821            A nullary function that returns sleep time in milliseconds
822        timeout : int, default=-1
823            Maximum time to wait in milliseconds. -1 means wait forever.
824
825        Returns
826        -------
827        Node
828            Result node
829
830        Raises
831        ------
832        TimeoutError
833            If the function call exceeds the timeout
834        Error
835            If the function returns an error
836        """
837        return self.dag.call(self, *args, name=name, sleep=sleep, timeout=timeout)

Representation of a node in a DaggerML DAG.

Parameters
  • dag (Dag): Parent DAG
  • ref (Ref): Node reference

Inherited members

Node
Node
dag
ref
context
type
value

Projection

@dataclass(frozen=True)
class Projection:
View source
840@dataclass(frozen=True)
841class Projection:
842    dag: Dag
843    base: Node
844    path: tuple[ProjectionStep, ...]
845    _info: dict = field(default_factory=dict)
846
847    def __repr__(self):
848        return f"Projection({self.base!r}, path={self.path!r})"
849
850    @classmethod
851    def from_step(cls, base: Node, step: ProjectionStep) -> "Projection":
852        return cls(dag=base.dag, base=base, path=(step,))
853
854    def _extend(self, step: ProjectionStep) -> "Projection":
855        return Projection(dag=self.dag, base=self.base, path=(*self.path, step))
856
857    def value(self):
858        return _apply_projection_path(self.base.value(), self.path)
859
860    def context(self, *, root: bool = True) -> Dag:
861        return _resolve_context(self.base, self.path, root=root)
862
863    def __call__(self, *args, **kwargs):
864        raise TypeError(f"Projection of type '{self.type}' is not callable")
865
866    @property
867    def type(self):
868        if "data_type" in self._info:
869            return self._info["data_type"]
870        return _info_for_value(self.value())["data_type"]
871
872    def __len__(self):
873        if "length" in self._info:
874            return self._info["length"]
875        value = self.value()
876        if not isinstance(value, (list, dict)):
877            raise TypeError(f"Object of type '{type(value).__name__}' has no len()")
878        return len(value)
879
880    def __iter__(self):
881        value = self.value()
882        if isinstance(value, list):
883            for i in range(len(value)):
884                yield self[i]
885            return
886        if isinstance(value, dict):
887            yield from self.keys()
888            return
889        raise TypeError(f"Object of type '{type(value).__name__}' is not iterable")
890
891    def __getitem__(self, key: ProjectionStep | slice) -> "Projection":
892        value = self.value()
893        if isinstance(value, dict):
894            if not isinstance(key, str):
895                raise TypeError(f"Dict keys must be strings but got {type(key).__name__}")
896            step = cast(ProjectionStep, key)
897        elif isinstance(value, list):
898            if not isinstance(key, (int, slice)):
899                raise TypeError(f"List indices must be integers or slices but got {type(key).__name__}")
900            step = _normalize_projection_step(cast(ProjectionStep | slice, key), length=len(value))
901        else:
902            raise TypeError(f"Cannot project into object of type '{type(value).__name__}'")
903        projected = self._extend(step)
904        return Projection(dag=self.dag, base=self.base, path=projected.path, _info=_info_for_value(projected.value()))
905
906    def keys(self) -> list[str]:
907        value = self.value()
908        if not isinstance(value, dict):
909            raise TypeError(f"Cannot get keys of type: {type(value).__name__}")
910        return sorted(value.keys())

Projection.__init__

Projection( dag: Dag, base: Node, path: tuple[typing.Union[str, int, list[int]], ...], _info: dict = <factory>)

Projection.dag

dag: Dag

Projection.base

base: Node

Projection.path

path: tuple[typing.Union[str, int, list[int]], ...]

Projection.from_step

@classmethod
def from_step( cls, base: Node, step: Union[str, int, list[int]]) -> Projection:
View source
850    @classmethod
851    def from_step(cls, base: Node, step: ProjectionStep) -> "Projection":
852        return cls(dag=base.dag, base=base, path=(step,))

Projection.value

def value(self):
View source
857    def value(self):
858        return _apply_projection_path(self.base.value(), self.path)

Projection.context

def context(self, *, root: bool = True) -> Dag:
View source
860    def context(self, *, root: bool = True) -> Dag:
861        return _resolve_context(self.base, self.path, root=root)

Projection.type

type
View source
866    @property
867    def type(self):
868        if "data_type" in self._info:
869            return self._info["data_type"]
870        return _info_for_value(self.value())["data_type"]

Projection.keys

def keys(self) -> list[str]:
View source
906    def keys(self) -> list[str]:
907        value = self.value()
908        if not isinstance(value, dict):
909            raise TypeError(f"Cannot get keys of type: {type(value).__name__}")
910        return sorted(value.keys())

CollectionNode

class CollectionNode(Node):
View source
913class CollectionNode(Node):  # noqa: F811
914    """
915    Representation of a collection node in a DaggerML DAG.
916
917    Parameters
918    ----------
919    dag : Dag
920        Parent DAG
921    ref : Ref
922        Node reference
923    """
924
925    def contains(self, item, *, name=None) -> "ScalarNode":
926        """
927        For collection nodes, checks to see if `item` is in `self`
928
929        Returns
930        -------
931        Node
932            Node with the boolean of is `item` in `self`
933        """
934        item_ref = item.ref if isinstance(item, Node) else item
935        result = self.dag._call_builtin("daggerml:contains", self.ref, item_ref, name=name)
936        return cast(ScalarNode, _make_node(self.dag, result))
937
938    def __contains__(self, item):
939        return self.contains(item).value()  # has to return boolean
940
941    def __len__(self):  # python requires this to be an int
942        """
943        Get the node's length
944
945        Returns
946        -------
947        Node
948            Node with the length of the collection
949
950        Raises
951        ------
952        Error
953            If the node isn't a collection (e.g. list or dict).
954        """
955        return self._info["length"]

Representation of a collection node in a DaggerML DAG.

Parameters
  • dag (Dag): Parent DAG
  • ref (Ref): Node reference

CollectionNode.contains

def contains(self, item, *, name=None) -> ScalarNode:
View source
925    def contains(self, item, *, name=None) -> "ScalarNode":
926        """
927        For collection nodes, checks to see if `item` is in `self`
928
929        Returns
930        -------
931        Node
932            Node with the boolean of is `item` in `self`
933        """
934        item_ref = item.ref if isinstance(item, Node) else item
935        result = self.dag._call_builtin("daggerml:contains", self.ref, item_ref, name=name)
936        return cast(ScalarNode, _make_node(self.dag, result))

For collection nodes, checks to see if item is in self

Returns
  • Node: Node with the boolean of is item in self

Inherited members

Node
Node
dag
ref
context
type
value

ListNode

class ListNode(CollectionNode):
View source
 958class ListNode(CollectionNode):  # noqa: F811
 959    """
 960    Representation of a collection node in a DaggerML DAG.
 961
 962    Parameters
 963    ----------
 964    dag : Dag
 965        Parent DAG
 966    ref : Ref
 967        Node reference
 968    """
 969
 970    @overload
 971    def __getitem__(self, key: Union[slice, list[int]]) -> Union["ListNode", Projection]: ...
 972    @overload
 973    def __getitem__(self, key: Union[int, "Node"]) -> Union["Node", Projection]: ...
 974    def __getitem__(self, key: Union[slice, list[int], int, "Node"]) -> Union["Node", Projection]:
 975        if self.dag.ref is not None:
 976            if isinstance(key, Node):
 977                raise TypeError("Committed list projections require concrete int or slice keys")
 978            step = _normalize_projection_step(key, length=len(self))
 979            return Projection(
 980                dag=self.dag,
 981                base=self,
 982                path=(step,),
 983                _info=_info_for_value(_apply_projection_step(self.value(), step)),
 984            )
 985        if isinstance(key, slice):
 986            if key.step is not None:
 987                raise ValueError("Slice step is not supported")
 988            start = key.start if key.start is not None else 0
 989            stop = key.stop if key.stop is not None else len(self)
 990            key = [start, stop]
 991        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key))
 992
 993    def __iter__(self):
 994        """
 995        Iterate over the node's values (items if it's a list, and keys if it's a
 996        dict)
 997
 998        Returns
 999        -------
1000        Node
1001            Result node
1002
1003        Raises
1004        ------
1005        Error
1006            If the node isn't a collection (e.g. list or dict).
1007        """
1008        for i in range(len(self)):
1009            yield self[i]
1010
1011    def conj(self, item, *, name=None) -> "ListNode":
1012        """
1013        For a list node, append an item
1014
1015        Returns
1016        -------
1017        Node
1018            Node containing the new collection
1019
1020        Notes
1021        -----
1022        `append` is an alias `conj`
1023        """
1024        item_ref = item.ref if isinstance(item, Node) else item
1025        resp = self.dag._call_builtin("daggerml:conj", self.ref, item_ref, name=name)
1026        return cast(ListNode, _make_node(self.dag, resp))
1027
1028    def append(self, item, *, name=None) -> "ListNode":
1029        """
1030        For a list node, append an item
1031
1032        Returns
1033        -------
1034        Node
1035            Node containing the new collection
1036
1037        See Also
1038        --------
1039        conj : The main implementation
1040        """
1041        return self.conj(item, name=name)

Representation of a collection node in a DaggerML DAG.

Parameters
  • dag (Dag): Parent DAG
  • ref (Ref): Node reference

ListNode.conj

def conj(self, item, *, name=None) -> ListNode:
View source
1011    def conj(self, item, *, name=None) -> "ListNode":
1012        """
1013        For a list node, append an item
1014
1015        Returns
1016        -------
1017        Node
1018            Node containing the new collection
1019
1020        Notes
1021        -----
1022        `append` is an alias `conj`
1023        """
1024        item_ref = item.ref if isinstance(item, Node) else item
1025        resp = self.dag._call_builtin("daggerml:conj", self.ref, item_ref, name=name)
1026        return cast(ListNode, _make_node(self.dag, resp))

For a list node, append an item

Returns
  • Node: Node containing the new collection
Notes

append is an alias conj

ListNode.append

def append(self, item, *, name=None) -> ListNode:
View source
1028    def append(self, item, *, name=None) -> "ListNode":
1029        """
1030        For a list node, append an item
1031
1032        Returns
1033        -------
1034        Node
1035            Node containing the new collection
1036
1037        See Also
1038        --------
1039        conj : The main implementation
1040        """
1041        return self.conj(item, name=name)

For a list node, append an item

Returns
  • Node: Node containing the new collection
See Also

conj: The main implementation

DictNode

class DictNode(CollectionNode):
View source
1044class DictNode(CollectionNode):  # noqa: F811
1045    def __getitem__(self, key: Union[str, "Node"]) -> Union["Node", Projection]:
1046        if self.dag.ref is not None:
1047            if not isinstance(key, str):
1048                raise TypeError(f"Dict keys must be strings but got {type(key).__name__}")
1049            return Projection(dag=self.dag, base=self, path=(key,), _info=_info_for_value(self.value()[key]))
1050        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key))
1051
1052    def keys(self) -> list[str]:
1053        """
1054        Get the keys of a dictionary node.
1055
1056        Parameters
1057        ----------
1058        name : str, optional
1059            Name for the result node
1060
1061        Returns
1062        -------
1063        list[str]
1064            List of keys in the dictionary node
1065        """
1066        return self._info["keys"].copy()
1067
1068    def __iter__(self):
1069        """
1070        Iterate over the node's values (items if it's a list, and keys if it's a
1071        dict)
1072
1073        Returns
1074        -------
1075        Node
1076            Result node
1077
1078        Raises
1079        ------
1080        Error
1081            If the node isn't a collection (e.g. list or dict).
1082        """
1083        for k in self.keys():
1084            yield k
1085
1086    def get(self, key, default=None, *, name=None) -> "Node":
1087        """
1088        For a dict node, return the value for key if key exists, else default.
1089
1090        If default is not given, it defaults to None, so that this method never raises a KeyError.
1091        """
1092        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key, default, name=name))
1093
1094    def items(self) -> Iterator[tuple[str, "Node|Projection"]]:
1095        """
1096        Iterate over key-value pairs of a dictionary node.
1097
1098        Returns
1099        -------
1100        Iterator[tuple[Node, Node]]
1101            Iterator over (key, value) pairs
1102        """
1103        if self.type != "dict":
1104            raise Error(f"Cannot iterate items of type: {self.type}", origin="dml", type="TypeError")
1105        for k in self:
1106            yield k, self[k]
1107
1108    def values(self) -> list["Node|Projection"]:
1109        """
1110        Get the values of a dictionary node.
1111
1112        Parameters
1113        ----------
1114        name : str, optional
1115            Name for the result node
1116
1117        Returns
1118        -------
1119        list[Node]
1120            List of values in the dictionary node
1121        """
1122        return [self[k] for k in self]
1123
1124    def assoc(self, key, value, *, name=None) -> "DictNode":
1125        """
1126        For a dict node, associate a new value into the map
1127
1128        Returns
1129        -------
1130        Node
1131            Node containing the new dict
1132        """
1133        value_ref = value.ref if isinstance(value, Node) else value
1134        resp = self.dag._call_builtin("daggerml:assoc", self.ref, key, value_ref, name=name)
1135        return cast(DictNode, _make_node(self.dag, resp))
1136
1137    def update(self, update) -> "DictNode":
1138        """
1139        For a dict node, update like python dicts
1140
1141        Returns
1142        -------
1143        Node
1144            Node containing the new collection
1145
1146        Notes
1147        -----
1148        calls `assoc` iteratively for k, v pairs in update.
1149
1150        See Also
1151        --------
1152        assoc : The main implementation
1153        """
1154        for k, v in update.items():
1155            self = self.assoc(k, v)
1156        return self

Representation of a collection node in a DaggerML DAG.

Parameters
  • dag (Dag): Parent DAG
  • ref (Ref): Node reference

DictNode.keys

def keys(self) -> list[str]:
View source
1052    def keys(self) -> list[str]:
1053        """
1054        Get the keys of a dictionary node.
1055
1056        Parameters
1057        ----------
1058        name : str, optional
1059            Name for the result node
1060
1061        Returns
1062        -------
1063        list[str]
1064            List of keys in the dictionary node
1065        """
1066        return self._info["keys"].copy()

Get the keys of a dictionary node.

Parameters
  • name (str, optional): Name for the result node
Returns
  • list[str]: List of keys in the dictionary node

DictNode.get

def get(self, key, default=None, *, name=None) -> Node:
View source
1086    def get(self, key, default=None, *, name=None) -> "Node":
1087        """
1088        For a dict node, return the value for key if key exists, else default.
1089
1090        If default is not given, it defaults to None, so that this method never raises a KeyError.
1091        """
1092        return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key, default, name=name))

For a dict node, return the value for key if key exists, else default.

If default is not given, it defaults to None, so that this method never raises a KeyError.

DictNode.items

def items( self) -> Iterator[tuple[str, Node | Projection]]:
View source
1094    def items(self) -> Iterator[tuple[str, "Node|Projection"]]:
1095        """
1096        Iterate over key-value pairs of a dictionary node.
1097
1098        Returns
1099        -------
1100        Iterator[tuple[Node, Node]]
1101            Iterator over (key, value) pairs
1102        """
1103        if self.type != "dict":
1104            raise Error(f"Cannot iterate items of type: {self.type}", origin="dml", type="TypeError")
1105        for k in self:
1106            yield k, self[k]

Iterate over key-value pairs of a dictionary node.

Returns
  • Iterator[tuple[Node, Node]]: Iterator over (key, value) pairs

DictNode.values

def values(self) -> list[Node | Projection]:
View source
1108    def values(self) -> list["Node|Projection"]:
1109        """
1110        Get the values of a dictionary node.
1111
1112        Parameters
1113        ----------
1114        name : str, optional
1115            Name for the result node
1116
1117        Returns
1118        -------
1119        list[Node]
1120            List of values in the dictionary node
1121        """
1122        return [self[k] for k in self]

Get the values of a dictionary node.

Parameters
  • name (str, optional): Name for the result node
Returns
  • list[Node]: List of values in the dictionary node

DictNode.assoc

def assoc(self, key, value, *, name=None) -> DictNode:
View source
1124    def assoc(self, key, value, *, name=None) -> "DictNode":
1125        """
1126        For a dict node, associate a new value into the map
1127
1128        Returns
1129        -------
1130        Node
1131            Node containing the new dict
1132        """
1133        value_ref = value.ref if isinstance(value, Node) else value
1134        resp = self.dag._call_builtin("daggerml:assoc", self.ref, key, value_ref, name=name)
1135        return cast(DictNode, _make_node(self.dag, resp))

For a dict node, associate a new value into the map

Returns
  • Node: Node containing the new dict

DictNode.update

def update(self, update) -> DictNode:
View source
1137    def update(self, update) -> "DictNode":
1138        """
1139        For a dict node, update like python dicts
1140
1141        Returns
1142        -------
1143        Node
1144            Node containing the new collection
1145
1146        Notes
1147        -----
1148        calls `assoc` iteratively for k, v pairs in update.
1149
1150        See Also
1151        --------
1152        assoc : The main implementation
1153        """
1154        for k, v in update.items():
1155            self = self.assoc(k, v)
1156        return self

For a dict node, update like python dicts

Returns
  • Node: Node containing the new collection
Notes

calls assoc iteratively for k, v pairs in update.

See Also

assoc: The main implementation

LITERAL_CODEC_ENTRYPOINT_GROUP

LITERAL_CODEC_ENTRYPOINT_GROUP= 'codecs'

LiteralCodec

class LiteralCodec(typing.Protocol):
View source
1169class LiteralCodec(Protocol):
1170    def can_encode(self, value: Any) -> bool: ...
1171
1172    def encode(self, value: Any, dag: "Dag") -> Any: ...

Base class for protocol classes.

Protocol classes are defined as::

class Proto(Protocol):
    def meth(self) -> int:
        ...

Such classes are primarily used with static type checkers that recognize structural subtyping (static duck-typing).

For example::

class C:
    def meth(self) -> int:
        return 0

def func(x: Proto) -> int:
    return x.meth()

func(C())  # Passes static type check

See PEP 544 for details. Protocol classes decorated with @typing.runtime_checkable act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as::

class GenProto[T](Protocol):
    def meth(self) -> T:
        ...

LiteralCodec.__init__

LiteralCodec(*args, **kwargs)
View source
1968def _no_init_or_replace_init(self, *args, **kwargs):
1969    cls = type(self)
1970
1971    if cls._is_protocol:
1972        raise TypeError('Protocols cannot be instantiated')
1973
1974    # Already using a custom `__init__`. No need to calculate correct
1975    # `__init__` to call. This can lead to RecursionError. See bpo-45121.
1976    if cls.__init__ is not _no_init_or_replace_init:
1977        return
1978
1979    # Initially, `__init__` of a protocol subclass is set to `_no_init_or_replace_init`.
1980    # The first instantiation of the subclass will call `_no_init_or_replace_init` which
1981    # searches for a proper new `__init__` in the MRO. The new `__init__`
1982    # replaces the subclass' old `__init__` (ie `_no_init_or_replace_init`). Subsequent
1983    # instantiation of the protocol subclass will thus use the new
1984    # `__init__` and no longer call `_no_init_or_replace_init`.
1985    for base in cls.__mro__:
1986        init = base.__dict__.get('__init__', _no_init_or_replace_init)
1987        if init is not _no_init_or_replace_init:
1988            cls.__init__ = init
1989            break
1990    else:
1991        # should not happen
1992        cls.__init__ = object.__init__
1993
1994    cls.__init__(self, *args, **kwargs)

LiteralCodec.can_encode

def can_encode(self, value: Any) -> bool:
View source
1170    def can_encode(self, value: Any) -> bool: ...

LiteralCodec.encode

def encode(self, value: Any, dag: Dag) -> Any:
View source
1172    def encode(self, value: Any, dag: "Dag") -> Any: ...

CodecError

class CodecError(daggerml._core.types.Error):
View source
1175class CodecError(Error):
1176    def __init__(self, message: str):
1177        super().__init__(message, origin="dml-codec", type="codec-error")

Error information with stack traces.

Represents a captured error from a computation, storing error details and stack trace information for debugging.

Attributes
  • message (str): The error message.
  • origin (str): The origin/source of the error (e.g., 'python', 'adapter').
  • type (str): The error type name.
  • stack (list[dict]): Stack trace frames as dictionaries.

CodecError.__init__

CodecError(message: str)
View source
1176    def __init__(self, message: str):
1177        super().__init__(message, origin="dml-codec", type="codec-error")

ensure_literal_codec_plugins_loaded

def ensure_literal_codec_plugins_loaded() -> None:
View source
1187def ensure_literal_codec_plugins_loaded() -> None:
1188    global _plugins_loaded
1189    if _plugins_loaded:
1190        return
1191    with _lock:
1192        codec_seq = 0
1193        if _plugins_loaded:
1194            return
1195        loaded = []
1196        for entry_point in _entry_points():
1197            try:
1198                registrations = entry_point.load()()
1199                for item in registrations:
1200                    priority, codec = item
1201                    codec_seq += 1
1202                    loaded.append((priority, codec_seq, codec))
1203            except Exception as e:
1204                msg = f"Literal codec plugin '{entry_point.name} ({entry_point.value})' failed: {e}"
1205                raise CodecError(msg) from None
1206        loaded.sort(key=lambda item: (-item[0], item[1]))
1207        _codecs.extend(loaded)
1208        _plugins_loaded = True

iter_codecs

def iter_codecs() -> Iterator[LiteralCodec]:
View source
1211def iter_codecs() -> Iterator[LiteralCodec]:
1212    ensure_literal_codec_plugins_loaded()
1213    yield from [codec for _priority, _seq, codec in _codecs]

apply_codec

def apply_codec(value: Any, *, dag: Dag) -> Any:
View source
1216def apply_codec(value: Any, *, dag: Dag) -> Any:
1217    for codec in iter_codecs():
1218        try:
1219            if codec.can_encode(value):
1220                resp = codec.encode(value, dag)
1221                if isinstance(resp, type(value)):
1222                    codec_name = codec.__class__.__name__
1223                    msg = f"Literal codec {codec_name} encoded {value.__class__.__name__} to {resp.__class__.__name__}."
1224                    raise CodecError(msg)
1225                return resp
1226        except Exception as e:
1227            if isinstance(e, DmlRepoError):
1228                raise
1229            raise CodecError(f"Literal codec {codec.__class__.__name__} failed: {e}") from e
1230    raise CodecError(f"No codec found for value of type {type(value).__name__}")

apply_codecs

def apply_codecs(value: Any, *, dag: Dag) -> Any:
View source
1233def apply_codecs(value: Any, *, dag: Dag) -> Any:
1234    while not isinstance(value, (*get_args(Scalar), *get_args(Collection), Ref)):
1235        value = apply_codec(value, dag=dag)
1236    if isinstance(value, list):
1237        return [apply_codecs(v, dag=dag) for v in value]
1238    if isinstance(value, dict):
1239        return {k: apply_codecs(v, dag=dag) for k, v in value.items()}
1240    if isinstance(value, Uri):
1241        return Uri(apply_codecs(value.uri, dag=dag))
1242    if isinstance(value, Runnable):
1243        target = apply_codecs(value.target, dag=dag)
1244        sub = apply_codecs(value.sub, dag=dag)
1245        kwargs = {k: apply_codecs(v, dag=dag) for k, v in value.kwargs.items()}
1246        return Runnable(target=target, adapter=value.adapter, kwargs=kwargs, sub=sub)
1247    return value

MiscPyTypeCodec

class MiscPyTypeCodec:
View source
1250class MiscPyTypeCodec:
1251    def can_encode(self, value: Any) -> bool:
1252        return isinstance(value, Mapping) or (
1253            isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray))
1254        )
1255
1256    def encode(self, value: Sequence | Mapping, dag: Dag) -> Any:
1257        if isinstance(value, Mapping):
1258            return {k: apply_codecs(v, dag=dag) for k, v in value.items()}
1259        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
1260            return [apply_codecs(v, dag=dag) for v in value]

MiscPyTypeCodec.can_encode

def can_encode(self, value: Any) -> bool:
View source
1251    def can_encode(self, value: Any) -> bool:
1252        return isinstance(value, Mapping) or (
1253            isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray))
1254        )

MiscPyTypeCodec.encode

def encode(self, value: Sequence | Mapping, dag: Dag) -> Any:
View source
1256    def encode(self, value: Sequence | Mapping, dag: Dag) -> Any:
1257        if isinstance(value, Mapping):
1258            return {k: apply_codecs(v, dag=dag) for k, v in value.items()}
1259        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
1260            return [apply_codecs(v, dag=dag) for v in value]

NodeCodec

class NodeCodec:
View source
1263class NodeCodec:
1264    def can_encode(self, value: Any) -> bool:
1265        return isinstance(value, Node)
1266
1267    def encode(self, value: "Node", dag: Dag) -> Ref:
1268        assert dag.token is not None, "DAG must have a token to encode nodes"
1269        if value.dag.token is not None and value.dag.token == dag.token:
1270            return value.ref
1271        if value.dag.ref is None:
1272            raise CodecError("Cannot encode node from uncommitted DAG in a different index")
1273        try:
1274            return dag.dml.runtime.put_import(dag._require_index_ref(), value.dag.ref, node=value.ref, name=None)
1275        except Exception as e:
1276            raise CodecError(f"Failed to encode cross-dag node import: {e}") from e

NodeCodec.can_encode

def can_encode(self, value: Any) -> bool:
View source
1264    def can_encode(self, value: Any) -> bool:
1265        return isinstance(value, Node)

NodeCodec.encode

def encode( self, value: Node, dag: Dag) -> daggerml.Ref:
View source
1267    def encode(self, value: "Node", dag: Dag) -> Ref:
1268        assert dag.token is not None, "DAG must have a token to encode nodes"
1269        if value.dag.token is not None and value.dag.token == dag.token:
1270            return value.ref
1271        if value.dag.ref is None:
1272            raise CodecError("Cannot encode node from uncommitted DAG in a different index")
1273        try:
1274            return dag.dml.runtime.put_import(dag._require_index_ref(), value.dag.ref, node=value.ref, name=None)
1275        except Exception as e:
1276            raise CodecError(f"Failed to encode cross-dag node import: {e}") from e

ProjectionCodec

class ProjectionCodec:
View source
1279class ProjectionCodec:
1280    def can_encode(self, value: Any) -> bool:
1281        return isinstance(value, Projection)
1282
1283    def encode(self, value: "Projection", dag: Dag) -> Ref:
1284        assert dag.token is not None, "DAG must have a token to encode projections"
1285        node_ref = NodeCodec().encode(value.base, dag)
1286        for step in value.path:
1287            node_ref = dag._call_builtin("daggerml:get", node_ref, step)
1288        return node_ref

ProjectionCodec.can_encode

def can_encode(self, value: Any) -> bool:
View source
1280    def can_encode(self, value: Any) -> bool:
1281        return isinstance(value, Projection)

ProjectionCodec.encode

def encode( self, value: Projection, dag: Dag) -> daggerml.Ref:
View source
1283    def encode(self, value: "Projection", dag: Dag) -> Ref:
1284        assert dag.token is not None, "DAG must have a token to encode projections"
1285        node_ref = NodeCodec().encode(value.base, dag)
1286        for step in value.path:
1287            node_ref = dag._call_builtin("daggerml:get", node_ref, step)
1288        return node_ref

codecs

def codecs() -> list:
View source
1291def codecs() -> list:
1292    return [(0, NodeCodec()), (0, MiscPyTypeCodec()), (0, ProjectionCodec())]