1from __future__ import annotations 2 3import logging 4import time 5import traceback 6from collections.abc import Iterator, Mapping, Sequence 7from contextlib import contextmanager 8from contextvars import ContextVar 9from dataclasses import dataclass, field 10from importlib import metadata 11from tempfile import TemporaryDirectory 12from threading import RLock 13from typing import Any, Optional, Protocol, Union, cast, get_args, overload 14 15from daggerml._core import CancellationError, Dml, DmlRepoError, Error, Ref, Runnable, Uri 16from daggerml.util import BackoffWithJitter, current_time_millis 17 18logger = logging.getLogger(__name__) 19 20Scalar = Union[str, int, float, bool, type(None), Uri, Runnable] 21Collection = Union[list, dict] 22ProjectionStep = Union[str, int, list[int]] 23 24_NO_DEFAULT_DML = object() 25_SCOPED_DEFAULT_DML: ContextVar[object] = ContextVar("daggerml_scoped_default_dml", default=_NO_DEFAULT_DML) 26_PROCESS_DEFAULT_DML: Optional["Dml"] = None 27 28 29def _resolve_default_dml(*, create: bool = True) -> tuple["Dml", str]: 30 scoped = _SCOPED_DEFAULT_DML.get() 31 if scoped is not _NO_DEFAULT_DML: 32 return cast("Dml", scoped), "scoped" 33 34 global _PROCESS_DEFAULT_DML 35 if _PROCESS_DEFAULT_DML is not None: 36 return _PROCESS_DEFAULT_DML, "process" 37 38 if not create: 39 raise DmlRepoError("No default Dml is configured") 40 41 _PROCESS_DEFAULT_DML = Dml() 42 return _PROCESS_DEFAULT_DML, "implicit" 43 44 45def get_default_dml() -> "Dml": 46 """Return the active default Dml runtime.""" 47 dml, _source = _resolve_default_dml(create=True) 48 return dml 49 50 51def set_default_dml(dml: "Dml") -> None: 52 """Set the process-default Dml runtime.""" 53 global _PROCESS_DEFAULT_DML 54 _PROCESS_DEFAULT_DML = dml 55 56 57def clear_default_dml() -> None: 58 """Clear the process-default Dml runtime.""" 59 global _PROCESS_DEFAULT_DML 60 _PROCESS_DEFAULT_DML = None 61 62 63@contextmanager 64def use_default_dml(dml: "Dml"): 65 """Temporarily override the default Dml runtime for the active context.""" 66 token = _SCOPED_DEFAULT_DML.set(dml) 67 try: 68 yield dml 69 finally: 70 _SCOPED_DEFAULT_DML.reset(token) 71 72 73def new( 74 name="", 75 message="", 76 cache_key: str | None = None, 77 execution_id: str | None = None, 78 tags: list[str] | None = None, 79 dml: Dml | None = None, 80) -> "Dag": 81 """Create a new DAG using the active or provided Dml runtime.""" 82 runtime = dml or get_default_dml() 83 execution = Ref(f"index:{execution_id}") if execution_id is not None else None 84 create_kwargs = {"cache_key": cache_key, "execution": execution} 85 if tags is not None: 86 create_kwargs["tags"] = tags 87 index_id = runtime.runtime.create(**create_kwargs) 88 return Dag(dml=runtime, token=index_id, name=name, message=message) 89 90 91def load( 92 name: str, 93 dml: Dml | None = None, 94 *, 95 revision: Ref | str = "HEAD", 96 remote: bool = False, 97 dep: str | None = None, 98) -> "Dag": 99 """Load a DAG using the active default Dml runtime.""" 100 dml = dml or get_default_dml() 101 dag_ref = dml.show(revision, remote=remote, dep=dep)["dags"].get(name) 102 if dag_ref is None: 103 raise DmlRepoError(f"DAG not found: {name}") 104 return Dag(dml=dml, ref=dag_ref, name=name) 105 106 107def resume( 108 frozen: Ref, 109 *, 110 name: str, 111 message: str, 112 dml: Dml | None = None, 113) -> "Dag": 114 """Resume a frozen DAG runtime with explicitly supplied commit metadata.""" 115 runtime = dml or get_default_dml() 116 index_id = runtime.runtime.unfreeze(frozen) 117 return Dag(dml=runtime, token=index_id, name=name, message=message) 118 119 120@contextmanager 121def temporary(prefix="dml-tmp-", **kw): 122 """Create a temporary Dml runtime with an unborn attached HEAD.""" 123 with TemporaryDirectory(prefix=prefix) as tmpdir: 124 yield Dml.init(project_home=tmpdir, **kw) 125 126 127def status() -> dict[str, object]: 128 """Return status for the active default Dml runtime.""" 129 dml, source = _resolve_default_dml(create=True) 130 return { 131 "default": { 132 "source": source, 133 "has_scoped_override": _SCOPED_DEFAULT_DML.get() is not _NO_DEFAULT_DML, 134 "has_process_default": _PROCESS_DEFAULT_DML is not None, 135 }, 136 "status": dml.status(), 137 } 138 139 140def _make_node(dag: "Dag", ref: Ref) -> "Node": 141 """ 142 Create a Node from a Dag and Ref. 143 144 Parameters 145 ---------- 146 dag : Dag 147 The parent DAG. 148 ref : Ref 149 The reference to the node. 150 Returns 151 ------- 152 Node 153 A Node instance representing the reference in the DAG. 154 """ 155 node_value = dag.dml.dag.get_node(ref) 156 if isinstance(node_value, Error): 157 raise NodeError(node_value, node_ref=ref, dag=dag) 158 info: dict[str, Any] = {"data_type": type(node_value).__name__.lower()} 159 # Determine node type based on value and populate info 160 if isinstance(node_value, list): 161 info["length"] = len(node_value) 162 node = ListNode(dag, ref, _info=info) 163 elif isinstance(node_value, dict): 164 info["length"] = len(node_value) 165 info["keys"] = sorted(node_value.keys()) 166 node = DictNode(dag, ref, _info=info) 167 elif isinstance(node_value, Runnable): 168 node = RunnableNode(dag, ref, _info=info) 169 else: 170 node = ScalarNode(dag, ref, _info=info) 171 return node 172 173 174def _info_for_value(value: Any) -> dict[str, Any]: 175 info: dict[str, Any] = {"data_type": type(value).__name__.lower()} 176 if isinstance(value, list): 177 info["length"] = len(value) 178 elif isinstance(value, dict): 179 info["length"] = len(value) 180 info["keys"] = sorted(value.keys()) 181 return info 182 183 184def _normalize_projection_step(key: ProjectionStep | slice, *, length: int | None = None) -> ProjectionStep: 185 if isinstance(key, slice): 186 if key.step is not None: 187 raise ValueError("Slice step is not supported") 188 start = key.start if key.start is not None else 0 189 if key.stop is None: 190 if length is None: 191 raise DmlRepoError("Slice stop requires known collection length") 192 stop = length 193 else: 194 stop = key.stop 195 return [start, stop] 196 return key 197 198 199def _apply_projection_step(value: Any, step: ProjectionStep) -> Any: 200 if isinstance(step, list): 201 if len(step) != 2: 202 raise DmlRepoError("Slice projection requires exactly [start, stop]") 203 return value[slice(*step)] 204 return value[step] 205 206 207def _apply_projection_path(value: Any, path: tuple[ProjectionStep, ...]) -> Any: 208 for step in path: 209 value = _apply_projection_step(value, step) 210 return value 211 212 213def _describe_node(node: "Node") -> Mapping[str, Any]: 214 return node.dag.dml.dag.describe_node(node.ref) 215 216 217def _builtin_name_for_argv(dag: "Dag", argv_refs: list[Ref]) -> str | None: 218 if not argv_refs: 219 return None 220 runnable = dag.dml.dag.get_node(argv_refs[0], recursive=True) 221 if not isinstance(runnable, Runnable): 222 return None 223 if runnable.adapter != "": 224 return None 225 if not runnable.target.uri.startswith("daggerml:"): 226 return None 227 return runnable.target.uri.split(":", 1)[1] 228 229 230def _prepend_get_path_step(path: tuple[ProjectionStep, ...], key: ProjectionStep) -> tuple[ProjectionStep, ...] | None: 231 if isinstance(key, list): 232 if len(key) != 2: 233 return None 234 if not path: 235 return None 236 first, *rest = path 237 if not isinstance(first, int): 238 return None 239 return (key[0] + first, *rest) 240 return (key, *path) 241 242 243def _backtrack_builtin( 244 node: "Node", argv_refs: list[Ref], path: tuple[ProjectionStep, ...] 245) -> tuple["Node", tuple[ProjectionStep, ...]] | None: 246 builtin = _builtin_name_for_argv(node.dag, argv_refs) 247 if builtin is None: 248 return None 249 arg_nodes = [_make_node(node.dag, ref) for ref in argv_refs[1:]] 250 if builtin == "get": 251 if len(arg_nodes) < 2: 252 return None 253 key = cast(ProjectionStep, arg_nodes[1].value()) 254 next_path = _prepend_get_path_step(path, key) 255 if next_path is None: 256 return None 257 return arg_nodes[0], next_path 258 if builtin == "list": 259 if not path: 260 return None 261 index, *rest = path 262 if not isinstance(index, int): 263 return None 264 if index < 0: 265 index += len(arg_nodes) 266 if index < 0 or index >= len(arg_nodes): 267 return None 268 return arg_nodes[index], tuple(rest) 269 if builtin == "dict": 270 if not path: 271 return None 272 key, *rest = path 273 if not isinstance(key, str): 274 return None 275 for idx in range(0, len(arg_nodes), 2): 276 if idx + 1 >= len(arg_nodes): 277 break 278 if arg_nodes[idx].value() == key: 279 return arg_nodes[idx + 1], tuple(rest) 280 return None 281 if builtin == "assoc": 282 if len(arg_nodes) < 3 or not path: 283 return None 284 selected_key, *rest = path 285 assoc_key = arg_nodes[1].value() 286 if selected_key == assoc_key: 287 return arg_nodes[2], tuple(rest) 288 return arg_nodes[0], path 289 if builtin == "conj": 290 if len(arg_nodes) < 2 or not path: 291 return None 292 index, *rest = path 293 if not isinstance(index, int): 294 return None 295 base_len = len(arg_nodes[0].value()) 296 if index < 0: 297 index += base_len + 1 298 if index == base_len: 299 return arg_nodes[1], tuple(rest) 300 return arg_nodes[0], path 301 return None 302 303 304def _nearest_context_state( 305 node: "Node", path: tuple[ProjectionStep, ...] 306) -> tuple[Dag, Node, tuple[ProjectionStep, ...], bool]: 307 current = node 308 current_path = path 309 while True: 310 node_info = _describe_node(current) 311 node_type = node_info["type"] 312 if node_type == "ImportNode": 313 source_dag = Dag(dml=current.dag.dml, ref=cast(Ref, node_info["dag"])) 314 source_node = _make_node(source_dag, cast(Ref, node_info["node"])) 315 if current_path: 316 current = source_node 317 continue 318 return source_dag, source_node, current_path, True 319 if node_type == "FnNode": 320 argv_refs = cast(list[Ref], node_info["argv"]) 321 source = _backtrack_builtin(current, argv_refs, current_path) 322 if source is not None: 323 current, current_path = source 324 continue 325 fn_dag = Dag(dml=current.dag.dml, ref=cast(Ref, node_info["dag"])) 326 return fn_dag, fn_dag.result, current_path, True 327 return current.dag, current, current_path, False 328 329 330def _resolve_context(node: "Node", path: tuple[ProjectionStep, ...], *, root: bool) -> Dag: 331 context_dag, next_node, next_path, can_recurse = _nearest_context_state(node, path) 332 if not root: 333 return context_dag 334 if not can_recurse: 335 return context_dag 336 return _resolve_context(next_node, next_path, root=True) 337 338 339@dataclass 340class Dag: 341 dml: Dml 342 token: Optional[Ref] = None # Working index id 343 ref: Optional[Ref] = None 344 name: str = "" # DAG name for commit 345 message: str = "" # Commit message 346 347 def __repr__(self): 348 to = self.ref.to if self.ref else (self.token.to if self.token is not None else "NA") 349 return f"Dag({to})" 350 351 def __hash__(self): 352 "Useful only for tests." 353 return 42 354 355 def __eq__(self, other): 356 "DAG equality is based on identity, not content." 357 if not isinstance(other, Dag): 358 return False 359 return self.ref == other.ref and self.token == other.token and self.dml == other.dml 360 361 def __enter__(self): 362 "Catch exceptions and commit an Error" 363 assert not self.ref 364 return self 365 366 def __exit__(self, exc_type, exc_value, tb): 367 if exc_value is not None: 368 # Convert exception to Error and commit it 369 traceback.print_exception(exc_type, exc_value, tb) 370 err = Error.from_ex(exc_value) if not isinstance(exc_value, Error) else exc_value 371 self.commit(err) 372 373 def _require_index_ref(self) -> Ref: 374 if self.token is None: 375 raise DmlRepoError("No active index") 376 return self.token 377 378 def _read_dag_ref(self) -> Ref: 379 """Return the completed or partial DAG backing this wrapper's reads.""" 380 if self.ref is not None: 381 return self.ref 382 return cast(Ref, self.dml.runtime.describe(self._require_index_ref())["dag"]) 383 384 @property 385 def tags(self) -> list[str]: 386 """Return the normalized tags stored on this DAG.""" 387 return list(self.dml.dag.describe(self._read_dag_ref())["tags"]) 388 389 def _put_literal(self, value: Any, *, name: Optional[str] = None) -> Ref: 390 index_id = self._require_index_ref() 391 value = apply_codecs(value, dag=self) 392 return self.dml.runtime.put_literal(index_id, value, name=name) 393 394 def _start_fn(self, argv: list[Ref], *, name: Optional[str] = None) -> Optional[Ref]: 395 return self.dml.runtime.start_fn(self._require_index_ref(), argv, name=name) 396 397 def _call_builtin(self, uri: str, *args: Any, name: Optional[str] = None) -> Ref: 398 fn_ref = self._put_literal(Runnable(target=Uri(uri), kwargs={}, adapter="")) 399 argv: list[Ref] = [fn_ref] 400 for arg in args: 401 argv.append(arg if isinstance(arg, Ref) else self._put_literal(arg)) 402 result = self._start_fn(argv, name=name) 403 if result is None: 404 raise DmlRepoError("Function execution failed") 405 return result 406 407 def __len__(self) -> int: 408 return len(self.keys()) 409 410 def __iter__(self): 411 yield from self.keys() 412 413 def _get_named_node(self, name: str) -> "Node": 414 node_ref = self.dml.dag.describe(self._read_dag_ref())["names"].get(name) 415 if node_ref is None: 416 raise DmlRepoError(f"Node '{name}' not found in DAG") 417 return _make_node(self, node_ref) 418 419 def _set_named_node(self, name: str, value: Any) -> None: 420 if self.ref is not None: 421 raise DmlRepoError("Cannot set node names on a committed DAG.") 422 if isinstance(value, Node): 423 value = value.ref 424 if isinstance(value, Ref): 425 self.dml.runtime.set_node_name(self._require_index_ref(), name, value) 426 return 427 self.put(value, name=name) 428 429 def __getitem__(self, name: str) -> "Node": 430 return self._get_named_node(name) 431 432 def __setitem__(self, name: str, value: Any) -> None: 433 self.put(value, name=name) 434 435 def __getattr__(self, name: str) -> "Node": 436 if name.startswith("_"): 437 raise AttributeError(name) 438 return self[name] 439 440 def __setattr__(self, name: str, value: Any) -> None: 441 dataclass_fields = getattr(type(self), "__dataclass_fields__", {}) 442 if name in dataclass_fields or hasattr(type(self), name): 443 object.__setattr__(self, name, value) 444 return 445 self[name] = value 446 447 def keys(self) -> list[str]: 448 """Get the list of all node names in the dag""" 449 names_dict = self.dml.dag.describe(self._read_dag_ref())["names"] 450 return sorted(names_dict.keys()) 451 452 def values(self) -> list["Node"]: 453 """Get the list of all nodes in the dag""" 454 names_dict = self.dml.dag.describe(self._read_dag_ref())["names"] 455 return [_make_node(self, ref) for ref in names_dict.values()] 456 457 @property 458 def argv(self) -> "ListNode": 459 "Access the dag's argv node" 460 argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"] 461 if not isinstance(argv_ref, Ref): 462 raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv") 463 return cast(ListNode, _make_node(self, argv_ref)) 464 465 @property 466 def result(self) -> "Node": 467 """Get the result node of the dag""" 468 if self.ref is None: 469 raise DmlRepoError("Cannot access result of an uncommitted DAG") 470 description = self.dml.dag.describe(self.ref) 471 error_ref = description.get("error") 472 if isinstance(error_ref, Ref): 473 raise self.dml.dag.get_error(error_ref) 474 ref = description.get("result") 475 if not isinstance(ref, Ref): 476 raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet") 477 return _make_node(self, ref) 478 479 @overload 480 def put(self, value: Union[list, "ListNode"], *, name=None) -> "ListNode": ... 481 @overload 482 def put(self, value: Union[dict, "DictNode"], *, name=None) -> "DictNode": ... 483 @overload 484 def put(self, value: Union[Runnable, "RunnableNode"], *, name=None) -> "RunnableNode": ... 485 @overload 486 def put(self, value: Union[Scalar, "ScalarNode"], *, name=None) -> "ScalarNode": ... 487 @overload 488 def put(self, value: Any, *, name=None) -> "Node": ... 489 def put(self, value: Any, *, name=None) -> "Node": 490 """ 491 Add a value to the DAG. 492 493 Parameters 494 ---------- 495 value : Union[Scalar, Collection] 496 Value to add 497 name : str, optional 498 Name for the node 499 Returns 500 ------- 501 Node 502 Node representing the value 503 504 Examples 505 -------- 506 >>> n1 = dag.put(42, name="answer") 507 >>> n1.value() 508 42 509 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}) 510 >>> n2.value() 511 {'a': 1, 'b': [42, '23']} 512 >>> n3 = dag.put({"a": 1, "b": [n1, "23"]}) 513 >>> n3.value() 514 {'a': 1, 'b': [42, '23']} 515 """ 516 return _make_node(self, self._put_literal(value, name=name)) 517 518 def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node": 519 """ 520 Import a node from a different (committed) DAG into the current DAG. 521 522 Parameters 523 ---------- 524 dag_name : str 525 Name of the DAG to import from 526 node_name : str, optional 527 Name of the node to import. If None, imports the result node of the DAG. 528 529 Returns 530 ------- 531 Node 532 The loaded node or DAG 533 534 Examples 535 -------- 536 >>> dag = new(dml=dml, name="test", message="test") 537 >>> n1 = dag.put(42, name="answer") 538 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data") 539 >>> dag.commit(n2) 540 >>> dag2 = new(dml=dml, name="test2", message="test2") 541 >>> imported_n2 = dag2.require("test", "data", name="imported_data") 542 >>> imported_n2.value() 543 {'a': 1, 'b': [42, '23']} 544 """ 545 index = self._require_index_ref() 546 if isinstance(dag_name, Dag): 547 if dag_name.ref is None: 548 raise DmlRepoError("Cannot import an uncommitted DAG") 549 dag_ref = dag_name.ref 550 else: 551 commit = self.dml.runtime.describe(index)["parents"][0] 552 dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name) 553 if dag_ref is None: 554 raise DmlRepoError(f"DAG not found: {dag_name}") 555 dag_info = self.dml.dag.describe(dag_ref) 556 node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result") 557 if node_ref is None: 558 raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'") 559 node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name) 560 return _make_node(self, node_ref) 561 562 def call( 563 self, 564 fn: Any, 565 *args: Any, 566 name: Optional[str] = None, 567 sleep: Optional[callable] = None, 568 timeout: int = -1, 569 ) -> "Node": 570 """ 571 Call a function node with arguments. 572 573 Parameters 574 ---------- 575 fn : Union[Runnable, RunnableNode] 576 Function to call 577 *args : Union[Node, Scalar, Collection] 578 Arguments to pass to the function 579 name : str, optional 580 Name for the result node 581 sleep : callable, optional 582 A nullary function that returns sleep time in milliseconds 583 timeout : int, default=-1 584 Maximum time to wait in milliseconds. If <= 0, wait indefinitely. 585 586 Returns 587 ------- 588 Node 589 Result node 590 591 Raises 592 ------ 593 TimeoutError 594 If the function call exceeds the timeout 595 Error 596 If the function returns an error 597 """ 598 sleep = sleep or BackoffWithJitter() 599 argv_seed = [fn, *args] 600 end = current_time_millis() + timeout 601 while timeout <= 0 or current_time_millis() < end: 602 argv_refs = [self._put_literal(value) for value in argv_seed] 603 resp = self._start_fn(argv_refs, name=name) 604 if resp: 605 return _make_node(self, resp) 606 time.sleep(sleep() / 1000) 607 raise TimeoutError(f"invoking function: {fn}") 608 609 def commit(self, value) -> None: 610 """ 611 Commit a value to the DAG. 612 613 Parameters 614 ---------- 615 value : Union[Node, Error, Any] 616 Value to commit 617 """ 618 # errors are committed as-is, everything else is a node 619 if not isinstance(value, (Error, Node)): 620 value = self.put(value) 621 if isinstance(value, Node): 622 value = value.ref 623 self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name) 624 self.token = None # Clear the working index since it's now committed 625 626 def freeze(self, message: str | None = None) -> "Dag": 627 """Freeze this uncommitted DAG's runtime index for read-only inspection.""" 628 freeze_message = f"dag: {self.name}" 629 if message: 630 freeze_message = f"{freeze_message}\n{message}" 631 self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message) 632 return self 633 634 def unfreeze(self) -> "Dag": 635 """Unfreeze this uncommitted DAG's runtime index.""" 636 self.token = self.dml.runtime.unfreeze(self._require_index_ref()) 637 return self 638 639 def dump(self) -> dict[str, Any]: 640 """Return this active or frozen runtime's repository objects as JSON-compatible data. 641 642 This does not freeze execution or preserve locks, execution records, or 643 external artifact bytes. ``dml.runtime.load(data)`` restores the original 644 index ID and overwrites an existing copy. 645 """ 646 return self.dml.runtime.dump(self._require_index_ref()) 647 648 @classmethod 649 def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag": 650 """Construct a DAG wrapper from a runtimepack, preserving its index ID. 651 652 Parameters 653 ---------- 654 data : dict 655 Runtimepack returned by ``Dag.dump()``. 656 dml : Dml, optional 657 Destination repository. Defaults to the active default repository. 658 659 Returns 660 ------- 661 Dag 662 The loaded active or frozen runtime. An existing index is overwritten; 663 execution records and locks are neither created nor restored. 664 """ 665 runtime = dml if dml is not None else get_default_dml() 666 return cls(dml=runtime, token=runtime.runtime.load(data)) 667 668 def cancel(self, max_retries: int = 3): 669 """Cancel the DAG's execution. 670 671 Parameters 672 ---------- 673 max_retries : int, default=3 674 Maximum cancellation retries after the initial attempt. 675 """ 676 if self.token is None: 677 raise DmlRepoError("Cannot cancel a committed DAG") 678 logger.info(f"Cancelling execution {self.token}") 679 self.dml.runtime.cancel(self.token, max_retries=max_retries) 680 self.token = None # Clear the index ref to indicate it's no longer active 681 raise CancellationError("DAG execution cancelled") 682 683 684class NodeError(Error): 685 """A stored error enriched with the node that produced it.""" 686 687 def __init__(self, error: Error, *, node_ref: Ref, dag: Dag): 688 super().__init__(message=error.message, origin=error.origin, type=error.type, stack=list(error.stack)) 689 self.node_ref = node_ref 690 self.dag = dag 691 692 def context(self) -> Dag: 693 """Return the function DAG that recorded this node's failure.""" 694 dag = self.dag 695 node_ref = self.node_ref 696 while True: 697 info = dag.dml.dag.describe_node(node_ref) 698 if info["type"] == "FnNode": 699 return Dag(dml=dag.dml, ref=info["dag"]) 700 if info["type"] != "ImportNode": 701 raise DmlRepoError(f"Node {node_ref} does not resolve to a failed function call") 702 dag = Dag(dml=dag.dml, ref=info["dag"]) 703 node_ref = info["node"] 704 705 706@dataclass(frozen=True) 707class Node: # noqa: F811 708 """ 709 Representation of a node in a DaggerML DAG. 710 711 Parameters 712 ---------- 713 dag : Dag 714 Parent DAG 715 ref : Ref 716 Node reference 717 """ 718 719 dag: Dag 720 ref: Ref 721 _info: dict = field(default_factory=dict) 722 723 def __repr__(self): 724 ref_id = self.ref if isinstance(self.ref, Error) else self.ref.to 725 return f"{self.__class__.__name__}({ref_id})" 726 727 def __hash__(self): 728 return hash(self.ref) 729 730 def __eq__(self, other): 731 if not isinstance(other, Node): 732 return NotImplemented 733 return self.ref == other.ref 734 735 def context(self, *, root: bool = True) -> Dag: 736 """ 737 Resolve the provenance context (DAG) for this node. 738 739 This follows import/function provenance while treating builtin 740 collection-construction and selection DAGs as transparent. 741 742 Parameters 743 ---------- 744 root : bool, default=True 745 If False, return the nearest sub-DAG in which this value exists as a 746 proper node across a non-builtin import/function boundary. If True, 747 continue recursively until provenance no longer crosses a 748 non-builtin import/function boundary and return that first rooted 749 context. 750 751 Returns 752 ------- 753 Dag 754 The nearest or rooted provenance DAG for this node. 755 756 Examples 757 -------- 758 >>> source = new(dml=dml, name="source", message="source") 759 >>> answer = source.put(42, name="answer") 760 >>> payload = source.put({"answer": answer}, name="payload") 761 >>> source.commit(payload) 762 >>> consumer = new(dml=dml, name="consumer", message="consumer") 763 >>> imported = consumer.require("source", "payload", name="payload") 764 >>> consumer.commit(imported) 765 >>> loaded = load("consumer", dml=dml) 766 >>> loaded.result["answer"].context() == source 767 True 768 """ 769 return _resolve_context(self, (), root=root) 770 771 @property 772 def type(self): 773 """Get the data type of the node.""" 774 return self._info["data_type"] 775 776 @overload 777 def value(self: "ScalarNode") -> Scalar: ... 778 @overload 779 def value(self: "ListNode") -> list: ... 780 @overload 781 def value(self: "DictNode") -> dict: ... 782 @overload 783 def value(self: "RunnableNode") -> Runnable: ... 784 @overload 785 def value(self: "Node") -> Any: ... 786 def value(self): 787 """ 788 Get the concrete value of this node. 789 790 Returns 791 ------- 792 Any 793 The actual value represented by this node 794 """ 795 value = self.dag.dml.dag.get_node(self.ref, recursive=True) 796 if isinstance(value, Error): 797 raise NodeError(value, node_ref=self.ref, dag=self.dag) 798 return value 799 800 def __call__(self, *args, name=None, sleep=None, timeout=-1, **kw) -> "Node": 801 raise TypeError(f"Node of type '{self.type}' is not callable") 802 803 804class ScalarNode(Node): 805 pass 806 807 808class RunnableNode(Node): 809 def __call__(self, *args, name=None, sleep=None, timeout=-1) -> "Node": 810 """ 811 Call this node as a function. 812 813 Parameters 814 ---------- 815 *args : Any 816 Arguments to pass to the function 817 name : str, optional 818 Name for the result node 819 sleep : callable, optional 820 A nullary function that returns sleep time in milliseconds 821 timeout : int, default=-1 822 Maximum time to wait in milliseconds. -1 means wait forever. 823 824 Returns 825 ------- 826 Node 827 Result node 828 829 Raises 830 ------ 831 TimeoutError 832 If the function call exceeds the timeout 833 Error 834 If the function returns an error 835 """ 836 return self.dag.call(self, *args, name=name, sleep=sleep, timeout=timeout) 837 838 839@dataclass(frozen=True) 840class Projection: 841 dag: Dag 842 base: Node 843 path: tuple[ProjectionStep, ...] 844 _info: dict = field(default_factory=dict) 845 846 def __repr__(self): 847 return f"Projection({self.base!r}, path={self.path!r})" 848 849 @classmethod 850 def from_step(cls, base: Node, step: ProjectionStep) -> "Projection": 851 return cls(dag=base.dag, base=base, path=(step,)) 852 853 def _extend(self, step: ProjectionStep) -> "Projection": 854 return Projection(dag=self.dag, base=self.base, path=(*self.path, step)) 855 856 def value(self): 857 return _apply_projection_path(self.base.value(), self.path) 858 859 def context(self, *, root: bool = True) -> Dag: 860 return _resolve_context(self.base, self.path, root=root) 861 862 def __call__(self, *args, **kwargs): 863 raise TypeError(f"Projection of type '{self.type}' is not callable") 864 865 @property 866 def type(self): 867 if "data_type" in self._info: 868 return self._info["data_type"] 869 return _info_for_value(self.value())["data_type"] 870 871 def __len__(self): 872 if "length" in self._info: 873 return self._info["length"] 874 value = self.value() 875 if not isinstance(value, (list, dict)): 876 raise TypeError(f"Object of type '{type(value).__name__}' has no len()") 877 return len(value) 878 879 def __iter__(self): 880 value = self.value() 881 if isinstance(value, list): 882 for i in range(len(value)): 883 yield self[i] 884 return 885 if isinstance(value, dict): 886 yield from self.keys() 887 return 888 raise TypeError(f"Object of type '{type(value).__name__}' is not iterable") 889 890 def __getitem__(self, key: ProjectionStep | slice) -> "Projection": 891 value = self.value() 892 if isinstance(value, dict): 893 if not isinstance(key, str): 894 raise TypeError(f"Dict keys must be strings but got {type(key).__name__}") 895 step = cast(ProjectionStep, key) 896 elif isinstance(value, list): 897 if not isinstance(key, (int, slice)): 898 raise TypeError(f"List indices must be integers or slices but got {type(key).__name__}") 899 step = _normalize_projection_step(cast(ProjectionStep | slice, key), length=len(value)) 900 else: 901 raise TypeError(f"Cannot project into object of type '{type(value).__name__}'") 902 projected = self._extend(step) 903 return Projection(dag=self.dag, base=self.base, path=projected.path, _info=_info_for_value(projected.value())) 904 905 def keys(self) -> list[str]: 906 value = self.value() 907 if not isinstance(value, dict): 908 raise TypeError(f"Cannot get keys of type: {type(value).__name__}") 909 return sorted(value.keys()) 910 911 912class CollectionNode(Node): # noqa: F811 913 """ 914 Representation of a collection node in a DaggerML DAG. 915 916 Parameters 917 ---------- 918 dag : Dag 919 Parent DAG 920 ref : Ref 921 Node reference 922 """ 923 924 def contains(self, item, *, name=None) -> "ScalarNode": 925 """ 926 For collection nodes, checks to see if `item` is in `self` 927 928 Returns 929 ------- 930 Node 931 Node with the boolean of is `item` in `self` 932 """ 933 item_ref = item.ref if isinstance(item, Node) else item 934 result = self.dag._call_builtin("daggerml:contains", self.ref, item_ref, name=name) 935 return cast(ScalarNode, _make_node(self.dag, result)) 936 937 def __contains__(self, item): 938 return self.contains(item).value() # has to return boolean 939 940 def __len__(self): # python requires this to be an int 941 """ 942 Get the node's length 943 944 Returns 945 ------- 946 Node 947 Node with the length of the collection 948 949 Raises 950 ------ 951 Error 952 If the node isn't a collection (e.g. list or dict). 953 """ 954 return self._info["length"] 955 956 957class ListNode(CollectionNode): # noqa: F811 958 """ 959 Representation of a collection node in a DaggerML DAG. 960 961 Parameters 962 ---------- 963 dag : Dag 964 Parent DAG 965 ref : Ref 966 Node reference 967 """ 968 969 @overload 970 def __getitem__(self, key: Union[slice, list[int]]) -> Union["ListNode", Projection]: ... 971 @overload 972 def __getitem__(self, key: Union[int, "Node"]) -> Union["Node", Projection]: ... 973 def __getitem__(self, key: Union[slice, list[int], int, "Node"]) -> Union["Node", Projection]: 974 if self.dag.ref is not None: 975 if isinstance(key, Node): 976 raise TypeError("Committed list projections require concrete int or slice keys") 977 step = _normalize_projection_step(key, length=len(self)) 978 return Projection( 979 dag=self.dag, 980 base=self, 981 path=(step,), 982 _info=_info_for_value(_apply_projection_step(self.value(), step)), 983 ) 984 if isinstance(key, slice): 985 if key.step is not None: 986 raise ValueError("Slice step is not supported") 987 start = key.start if key.start is not None else 0 988 stop = key.stop if key.stop is not None else len(self) 989 key = [start, stop] 990 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key)) 991 992 def __iter__(self): 993 """ 994 Iterate over the node's values (items if it's a list, and keys if it's a 995 dict) 996 997 Returns 998 ------- 999 Node 1000 Result node 1001 1002 Raises 1003 ------ 1004 Error 1005 If the node isn't a collection (e.g. list or dict). 1006 """ 1007 for i in range(len(self)): 1008 yield self[i] 1009 1010 def conj(self, item, *, name=None) -> "ListNode": 1011 """ 1012 For a list node, append an item 1013 1014 Returns 1015 ------- 1016 Node 1017 Node containing the new collection 1018 1019 Notes 1020 ----- 1021 `append` is an alias `conj` 1022 """ 1023 item_ref = item.ref if isinstance(item, Node) else item 1024 resp = self.dag._call_builtin("daggerml:conj", self.ref, item_ref, name=name) 1025 return cast(ListNode, _make_node(self.dag, resp)) 1026 1027 def append(self, item, *, name=None) -> "ListNode": 1028 """ 1029 For a list node, append an item 1030 1031 Returns 1032 ------- 1033 Node 1034 Node containing the new collection 1035 1036 See Also 1037 -------- 1038 conj : The main implementation 1039 """ 1040 return self.conj(item, name=name) 1041 1042 1043class DictNode(CollectionNode): # noqa: F811 1044 def __getitem__(self, key: Union[str, "Node"]) -> Union["Node", Projection]: 1045 if self.dag.ref is not None: 1046 if not isinstance(key, str): 1047 raise TypeError(f"Dict keys must be strings but got {type(key).__name__}") 1048 return Projection(dag=self.dag, base=self, path=(key,), _info=_info_for_value(self.value()[key])) 1049 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key)) 1050 1051 def keys(self) -> list[str]: 1052 """ 1053 Get the keys of a dictionary node. 1054 1055 Parameters 1056 ---------- 1057 name : str, optional 1058 Name for the result node 1059 1060 Returns 1061 ------- 1062 list[str] 1063 List of keys in the dictionary node 1064 """ 1065 return self._info["keys"].copy() 1066 1067 def __iter__(self): 1068 """ 1069 Iterate over the node's values (items if it's a list, and keys if it's a 1070 dict) 1071 1072 Returns 1073 ------- 1074 Node 1075 Result node 1076 1077 Raises 1078 ------ 1079 Error 1080 If the node isn't a collection (e.g. list or dict). 1081 """ 1082 for k in self.keys(): 1083 yield k 1084 1085 def get(self, key, default=None, *, name=None) -> "Node": 1086 """ 1087 For a dict node, return the value for key if key exists, else default. 1088 1089 If default is not given, it defaults to None, so that this method never raises a KeyError. 1090 """ 1091 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key, default, name=name)) 1092 1093 def items(self) -> Iterator[tuple[str, "Node|Projection"]]: 1094 """ 1095 Iterate over key-value pairs of a dictionary node. 1096 1097 Returns 1098 ------- 1099 Iterator[tuple[Node, Node]] 1100 Iterator over (key, value) pairs 1101 """ 1102 if self.type != "dict": 1103 raise Error(f"Cannot iterate items of type: {self.type}", origin="dml", type="TypeError") 1104 for k in self: 1105 yield k, self[k] 1106 1107 def values(self) -> list["Node|Projection"]: 1108 """ 1109 Get the values of a dictionary node. 1110 1111 Parameters 1112 ---------- 1113 name : str, optional 1114 Name for the result node 1115 1116 Returns 1117 ------- 1118 list[Node] 1119 List of values in the dictionary node 1120 """ 1121 return [self[k] for k in self] 1122 1123 def assoc(self, key, value, *, name=None) -> "DictNode": 1124 """ 1125 For a dict node, associate a new value into the map 1126 1127 Returns 1128 ------- 1129 Node 1130 Node containing the new dict 1131 """ 1132 value_ref = value.ref if isinstance(value, Node) else value 1133 resp = self.dag._call_builtin("daggerml:assoc", self.ref, key, value_ref, name=name) 1134 return cast(DictNode, _make_node(self.dag, resp)) 1135 1136 def update(self, update) -> "DictNode": 1137 """ 1138 For a dict node, update like python dicts 1139 1140 Returns 1141 ------- 1142 Node 1143 Node containing the new collection 1144 1145 Notes 1146 ----- 1147 calls `assoc` iteratively for k, v pairs in update. 1148 1149 See Also 1150 -------- 1151 assoc : The main implementation 1152 """ 1153 for k, v in update.items(): 1154 self = self.assoc(k, v) 1155 return self 1156 1157 1158################################################################################ 1159################## Codec system for encoding literals in DAGs ################## 1160################################################################################ 1161 1162LITERAL_CODEC_ENTRYPOINT_GROUP = "daggerml.codecs" 1163_codecs: list[tuple[int, int, "LiteralCodec"]] = [] 1164_plugins_loaded = False 1165_lock = RLock() 1166 1167 1168class LiteralCodec(Protocol): 1169 def can_encode(self, value: Any) -> bool: ... 1170 1171 def encode(self, value: Any, dag: "Dag") -> Any: ... 1172 1173 1174class CodecError(Error): 1175 def __init__(self, message: str): 1176 super().__init__(message, origin="dml-codec", type="codec-error") 1177 1178 1179def _entry_points(group=LITERAL_CODEC_ENTRYPOINT_GROUP) -> list[metadata.EntryPoint]: 1180 points = metadata.entry_points() 1181 result = list(points.select(group=group)) 1182 result.sort(key=lambda ep: (ep.name, ep.value)) 1183 return result 1184 1185 1186def ensure_literal_codec_plugins_loaded() -> None: 1187 global _plugins_loaded 1188 if _plugins_loaded: 1189 return 1190 with _lock: 1191 codec_seq = 0 1192 if _plugins_loaded: 1193 return 1194 loaded = [] 1195 for entry_point in _entry_points(): 1196 try: 1197 registrations = entry_point.load()() 1198 for item in registrations: 1199 priority, codec = item 1200 codec_seq += 1 1201 loaded.append((priority, codec_seq, codec)) 1202 except Exception as e: 1203 msg = f"Literal codec plugin '{entry_point.name} ({entry_point.value})' failed: {e}" 1204 raise CodecError(msg) from None 1205 loaded.sort(key=lambda item: (-item[0], item[1])) 1206 _codecs.extend(loaded) 1207 _plugins_loaded = True 1208 1209 1210def iter_codecs() -> Iterator[LiteralCodec]: 1211 ensure_literal_codec_plugins_loaded() 1212 yield from [codec for _priority, _seq, codec in _codecs] 1213 1214 1215def apply_codec(value: Any, *, dag: Dag) -> Any: 1216 for codec in iter_codecs(): 1217 try: 1218 if codec.can_encode(value): 1219 resp = codec.encode(value, dag) 1220 if isinstance(resp, type(value)): 1221 codec_name = codec.__class__.__name__ 1222 msg = f"Literal codec {codec_name} encoded {value.__class__.__name__} to {resp.__class__.__name__}." 1223 raise CodecError(msg) 1224 return resp 1225 except Exception as e: 1226 if isinstance(e, DmlRepoError): 1227 raise 1228 raise CodecError(f"Literal codec {codec.__class__.__name__} failed: {e}") from e 1229 raise CodecError(f"No codec found for value of type {type(value).__name__}") 1230 1231 1232def apply_codecs(value: Any, *, dag: Dag) -> Any: 1233 while not isinstance(value, (*get_args(Scalar), *get_args(Collection), Ref)): 1234 value = apply_codec(value, dag=dag) 1235 if isinstance(value, list): 1236 return [apply_codecs(v, dag=dag) for v in value] 1237 if isinstance(value, dict): 1238 return {k: apply_codecs(v, dag=dag) for k, v in value.items()} 1239 if isinstance(value, Uri): 1240 return Uri(apply_codecs(value.uri, dag=dag)) 1241 if isinstance(value, Runnable): 1242 target = apply_codecs(value.target, dag=dag) 1243 sub = apply_codecs(value.sub, dag=dag) 1244 kwargs = {k: apply_codecs(v, dag=dag) for k, v in value.kwargs.items()} 1245 return Runnable(target=target, adapter=value.adapter, kwargs=kwargs, sub=sub) 1246 return value 1247 1248 1249class MiscPyTypeCodec: 1250 def can_encode(self, value: Any) -> bool: 1251 return isinstance(value, Mapping) or ( 1252 isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) 1253 ) 1254 1255 def encode(self, value: Sequence | Mapping, dag: Dag) -> Any: 1256 if isinstance(value, Mapping): 1257 return {k: apply_codecs(v, dag=dag) for k, v in value.items()} 1258 if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): 1259 return [apply_codecs(v, dag=dag) for v in value] 1260 1261 1262class NodeCodec: 1263 def can_encode(self, value: Any) -> bool: 1264 return isinstance(value, Node) 1265 1266 def encode(self, value: "Node", dag: Dag) -> Ref: 1267 assert dag.token is not None, "DAG must have a token to encode nodes" 1268 if value.dag.token is not None and value.dag.token == dag.token: 1269 return value.ref 1270 if value.dag.ref is None: 1271 raise CodecError("Cannot encode node from uncommitted DAG in a different index") 1272 try: 1273 return dag.dml.runtime.put_import(dag._require_index_ref(), value.dag.ref, node=value.ref, name=None) 1274 except Exception as e: 1275 raise CodecError(f"Failed to encode cross-dag node import: {e}") from e 1276 1277 1278class ProjectionCodec: 1279 def can_encode(self, value: Any) -> bool: 1280 return isinstance(value, Projection) 1281 1282 def encode(self, value: "Projection", dag: Dag) -> Ref: 1283 assert dag.token is not None, "DAG must have a token to encode projections" 1284 node_ref = NodeCodec().encode(value.base, dag) 1285 for step in value.path: 1286 node_ref = dag._call_builtin("daggerml:get", node_ref, step) 1287 return node_ref 1288 1289 1290def codecs() -> list: 1291 return [(0, NodeCodec()), (0, MiscPyTypeCodec()), (0, ProjectionCodec())]
Return the active default Dml runtime.
Set the process-default Dml runtime.
Clear the process-default Dml runtime.
Temporarily override the default Dml runtime for the active context.
74def new( 75 name="", 76 message="", 77 cache_key: str | None = None, 78 execution_id: str | None = None, 79 tags: list[str] | None = None, 80 dml: Dml | None = None, 81) -> "Dag": 82 """Create a new DAG using the active or provided Dml runtime.""" 83 runtime = dml or get_default_dml() 84 execution = Ref(f"index:{execution_id}") if execution_id is not None else None 85 create_kwargs = {"cache_key": cache_key, "execution": execution} 86 if tags is not None: 87 create_kwargs["tags"] = tags 88 index_id = runtime.runtime.create(**create_kwargs) 89 return Dag(dml=runtime, token=index_id, name=name, message=message)
Create a new DAG using the active or provided Dml runtime.
92def load( 93 name: str, 94 dml: Dml | None = None, 95 *, 96 revision: Ref | str = "HEAD", 97 remote: bool = False, 98 dep: str | None = None, 99) -> "Dag": 100 """Load a DAG using the active default Dml runtime.""" 101 dml = dml or get_default_dml() 102 dag_ref = dml.show(revision, remote=remote, dep=dep)["dags"].get(name) 103 if dag_ref is None: 104 raise DmlRepoError(f"DAG not found: {name}") 105 return Dag(dml=dml, ref=dag_ref, name=name)
Load a DAG using the active default Dml runtime.
108def resume( 109 frozen: Ref, 110 *, 111 name: str, 112 message: str, 113 dml: Dml | None = None, 114) -> "Dag": 115 """Resume a frozen DAG runtime with explicitly supplied commit metadata.""" 116 runtime = dml or get_default_dml() 117 index_id = runtime.runtime.unfreeze(frozen) 118 return Dag(dml=runtime, token=index_id, name=name, message=message)
Resume a frozen DAG runtime with explicitly supplied commit metadata.
Create a temporary Dml runtime with an unborn attached HEAD.
128def status() -> dict[str, object]: 129 """Return status for the active default Dml runtime.""" 130 dml, source = _resolve_default_dml(create=True) 131 return { 132 "default": { 133 "source": source, 134 "has_scoped_override": _SCOPED_DEFAULT_DML.get() is not _NO_DEFAULT_DML, 135 "has_process_default": _PROCESS_DEFAULT_DML is not None, 136 }, 137 "status": dml.status(), 138 }
Return status for the active default Dml runtime.
340@dataclass 341class Dag: 342 dml: Dml 343 token: Optional[Ref] = None # Working index id 344 ref: Optional[Ref] = None 345 name: str = "" # DAG name for commit 346 message: str = "" # Commit message 347 348 def __repr__(self): 349 to = self.ref.to if self.ref else (self.token.to if self.token is not None else "NA") 350 return f"Dag({to})" 351 352 def __hash__(self): 353 "Useful only for tests." 354 return 42 355 356 def __eq__(self, other): 357 "DAG equality is based on identity, not content." 358 if not isinstance(other, Dag): 359 return False 360 return self.ref == other.ref and self.token == other.token and self.dml == other.dml 361 362 def __enter__(self): 363 "Catch exceptions and commit an Error" 364 assert not self.ref 365 return self 366 367 def __exit__(self, exc_type, exc_value, tb): 368 if exc_value is not None: 369 # Convert exception to Error and commit it 370 traceback.print_exception(exc_type, exc_value, tb) 371 err = Error.from_ex(exc_value) if not isinstance(exc_value, Error) else exc_value 372 self.commit(err) 373 374 def _require_index_ref(self) -> Ref: 375 if self.token is None: 376 raise DmlRepoError("No active index") 377 return self.token 378 379 def _read_dag_ref(self) -> Ref: 380 """Return the completed or partial DAG backing this wrapper's reads.""" 381 if self.ref is not None: 382 return self.ref 383 return cast(Ref, self.dml.runtime.describe(self._require_index_ref())["dag"]) 384 385 @property 386 def tags(self) -> list[str]: 387 """Return the normalized tags stored on this DAG.""" 388 return list(self.dml.dag.describe(self._read_dag_ref())["tags"]) 389 390 def _put_literal(self, value: Any, *, name: Optional[str] = None) -> Ref: 391 index_id = self._require_index_ref() 392 value = apply_codecs(value, dag=self) 393 return self.dml.runtime.put_literal(index_id, value, name=name) 394 395 def _start_fn(self, argv: list[Ref], *, name: Optional[str] = None) -> Optional[Ref]: 396 return self.dml.runtime.start_fn(self._require_index_ref(), argv, name=name) 397 398 def _call_builtin(self, uri: str, *args: Any, name: Optional[str] = None) -> Ref: 399 fn_ref = self._put_literal(Runnable(target=Uri(uri), kwargs={}, adapter="")) 400 argv: list[Ref] = [fn_ref] 401 for arg in args: 402 argv.append(arg if isinstance(arg, Ref) else self._put_literal(arg)) 403 result = self._start_fn(argv, name=name) 404 if result is None: 405 raise DmlRepoError("Function execution failed") 406 return result 407 408 def __len__(self) -> int: 409 return len(self.keys()) 410 411 def __iter__(self): 412 yield from self.keys() 413 414 def _get_named_node(self, name: str) -> "Node": 415 node_ref = self.dml.dag.describe(self._read_dag_ref())["names"].get(name) 416 if node_ref is None: 417 raise DmlRepoError(f"Node '{name}' not found in DAG") 418 return _make_node(self, node_ref) 419 420 def _set_named_node(self, name: str, value: Any) -> None: 421 if self.ref is not None: 422 raise DmlRepoError("Cannot set node names on a committed DAG.") 423 if isinstance(value, Node): 424 value = value.ref 425 if isinstance(value, Ref): 426 self.dml.runtime.set_node_name(self._require_index_ref(), name, value) 427 return 428 self.put(value, name=name) 429 430 def __getitem__(self, name: str) -> "Node": 431 return self._get_named_node(name) 432 433 def __setitem__(self, name: str, value: Any) -> None: 434 self.put(value, name=name) 435 436 def __getattr__(self, name: str) -> "Node": 437 if name.startswith("_"): 438 raise AttributeError(name) 439 return self[name] 440 441 def __setattr__(self, name: str, value: Any) -> None: 442 dataclass_fields = getattr(type(self), "__dataclass_fields__", {}) 443 if name in dataclass_fields or hasattr(type(self), name): 444 object.__setattr__(self, name, value) 445 return 446 self[name] = value 447 448 def keys(self) -> list[str]: 449 """Get the list of all node names in the dag""" 450 names_dict = self.dml.dag.describe(self._read_dag_ref())["names"] 451 return sorted(names_dict.keys()) 452 453 def values(self) -> list["Node"]: 454 """Get the list of all nodes in the dag""" 455 names_dict = self.dml.dag.describe(self._read_dag_ref())["names"] 456 return [_make_node(self, ref) for ref in names_dict.values()] 457 458 @property 459 def argv(self) -> "ListNode": 460 "Access the dag's argv node" 461 argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"] 462 if not isinstance(argv_ref, Ref): 463 raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv") 464 return cast(ListNode, _make_node(self, argv_ref)) 465 466 @property 467 def result(self) -> "Node": 468 """Get the result node of the dag""" 469 if self.ref is None: 470 raise DmlRepoError("Cannot access result of an uncommitted DAG") 471 description = self.dml.dag.describe(self.ref) 472 error_ref = description.get("error") 473 if isinstance(error_ref, Ref): 474 raise self.dml.dag.get_error(error_ref) 475 ref = description.get("result") 476 if not isinstance(ref, Ref): 477 raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet") 478 return _make_node(self, ref) 479 480 @overload 481 def put(self, value: Union[list, "ListNode"], *, name=None) -> "ListNode": ... 482 @overload 483 def put(self, value: Union[dict, "DictNode"], *, name=None) -> "DictNode": ... 484 @overload 485 def put(self, value: Union[Runnable, "RunnableNode"], *, name=None) -> "RunnableNode": ... 486 @overload 487 def put(self, value: Union[Scalar, "ScalarNode"], *, name=None) -> "ScalarNode": ... 488 @overload 489 def put(self, value: Any, *, name=None) -> "Node": ... 490 def put(self, value: Any, *, name=None) -> "Node": 491 """ 492 Add a value to the DAG. 493 494 Parameters 495 ---------- 496 value : Union[Scalar, Collection] 497 Value to add 498 name : str, optional 499 Name for the node 500 Returns 501 ------- 502 Node 503 Node representing the value 504 505 Examples 506 -------- 507 >>> n1 = dag.put(42, name="answer") 508 >>> n1.value() 509 42 510 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}) 511 >>> n2.value() 512 {'a': 1, 'b': [42, '23']} 513 >>> n3 = dag.put({"a": 1, "b": [n1, "23"]}) 514 >>> n3.value() 515 {'a': 1, 'b': [42, '23']} 516 """ 517 return _make_node(self, self._put_literal(value, name=name)) 518 519 def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node": 520 """ 521 Import a node from a different (committed) DAG into the current DAG. 522 523 Parameters 524 ---------- 525 dag_name : str 526 Name of the DAG to import from 527 node_name : str, optional 528 Name of the node to import. If None, imports the result node of the DAG. 529 530 Returns 531 ------- 532 Node 533 The loaded node or DAG 534 535 Examples 536 -------- 537 >>> dag = new(dml=dml, name="test", message="test") 538 >>> n1 = dag.put(42, name="answer") 539 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data") 540 >>> dag.commit(n2) 541 >>> dag2 = new(dml=dml, name="test2", message="test2") 542 >>> imported_n2 = dag2.require("test", "data", name="imported_data") 543 >>> imported_n2.value() 544 {'a': 1, 'b': [42, '23']} 545 """ 546 index = self._require_index_ref() 547 if isinstance(dag_name, Dag): 548 if dag_name.ref is None: 549 raise DmlRepoError("Cannot import an uncommitted DAG") 550 dag_ref = dag_name.ref 551 else: 552 commit = self.dml.runtime.describe(index)["parents"][0] 553 dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name) 554 if dag_ref is None: 555 raise DmlRepoError(f"DAG not found: {dag_name}") 556 dag_info = self.dml.dag.describe(dag_ref) 557 node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result") 558 if node_ref is None: 559 raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'") 560 node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name) 561 return _make_node(self, node_ref) 562 563 def call( 564 self, 565 fn: Any, 566 *args: Any, 567 name: Optional[str] = None, 568 sleep: Optional[callable] = None, 569 timeout: int = -1, 570 ) -> "Node": 571 """ 572 Call a function node with arguments. 573 574 Parameters 575 ---------- 576 fn : Union[Runnable, RunnableNode] 577 Function to call 578 *args : Union[Node, Scalar, Collection] 579 Arguments to pass to the function 580 name : str, optional 581 Name for the result node 582 sleep : callable, optional 583 A nullary function that returns sleep time in milliseconds 584 timeout : int, default=-1 585 Maximum time to wait in milliseconds. If <= 0, wait indefinitely. 586 587 Returns 588 ------- 589 Node 590 Result node 591 592 Raises 593 ------ 594 TimeoutError 595 If the function call exceeds the timeout 596 Error 597 If the function returns an error 598 """ 599 sleep = sleep or BackoffWithJitter() 600 argv_seed = [fn, *args] 601 end = current_time_millis() + timeout 602 while timeout <= 0 or current_time_millis() < end: 603 argv_refs = [self._put_literal(value) for value in argv_seed] 604 resp = self._start_fn(argv_refs, name=name) 605 if resp: 606 return _make_node(self, resp) 607 time.sleep(sleep() / 1000) 608 raise TimeoutError(f"invoking function: {fn}") 609 610 def commit(self, value) -> None: 611 """ 612 Commit a value to the DAG. 613 614 Parameters 615 ---------- 616 value : Union[Node, Error, Any] 617 Value to commit 618 """ 619 # errors are committed as-is, everything else is a node 620 if not isinstance(value, (Error, Node)): 621 value = self.put(value) 622 if isinstance(value, Node): 623 value = value.ref 624 self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name) 625 self.token = None # Clear the working index since it's now committed 626 627 def freeze(self, message: str | None = None) -> "Dag": 628 """Freeze this uncommitted DAG's runtime index for read-only inspection.""" 629 freeze_message = f"dag: {self.name}" 630 if message: 631 freeze_message = f"{freeze_message}\n{message}" 632 self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message) 633 return self 634 635 def unfreeze(self) -> "Dag": 636 """Unfreeze this uncommitted DAG's runtime index.""" 637 self.token = self.dml.runtime.unfreeze(self._require_index_ref()) 638 return self 639 640 def dump(self) -> dict[str, Any]: 641 """Return this active or frozen runtime's repository objects as JSON-compatible data. 642 643 This does not freeze execution or preserve locks, execution records, or 644 external artifact bytes. ``dml.runtime.load(data)`` restores the original 645 index ID and overwrites an existing copy. 646 """ 647 return self.dml.runtime.dump(self._require_index_ref()) 648 649 @classmethod 650 def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag": 651 """Construct a DAG wrapper from a runtimepack, preserving its index ID. 652 653 Parameters 654 ---------- 655 data : dict 656 Runtimepack returned by ``Dag.dump()``. 657 dml : Dml, optional 658 Destination repository. Defaults to the active default repository. 659 660 Returns 661 ------- 662 Dag 663 The loaded active or frozen runtime. An existing index is overwritten; 664 execution records and locks are neither created nor restored. 665 """ 666 runtime = dml if dml is not None else get_default_dml() 667 return cls(dml=runtime, token=runtime.runtime.load(data)) 668 669 def cancel(self, max_retries: int = 3): 670 """Cancel the DAG's execution. 671 672 Parameters 673 ---------- 674 max_retries : int, default=3 675 Maximum cancellation retries after the initial attempt. 676 """ 677 if self.token is None: 678 raise DmlRepoError("Cannot cancel a committed DAG") 679 logger.info(f"Cancelling execution {self.token}") 680 self.dml.runtime.cancel(self.token, max_retries=max_retries) 681 self.token = None # Clear the index ref to indicate it's no longer active 682 raise CancellationError("DAG execution cancelled")
458 @property 459 def argv(self) -> "ListNode": 460 "Access the dag's argv node" 461 argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"] 462 if not isinstance(argv_ref, Ref): 463 raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv") 464 return cast(ListNode, _make_node(self, argv_ref))
Access the dag's argv node
466 @property 467 def result(self) -> "Node": 468 """Get the result node of the dag""" 469 if self.ref is None: 470 raise DmlRepoError("Cannot access result of an uncommitted DAG") 471 description = self.dml.dag.describe(self.ref) 472 error_ref = description.get("error") 473 if isinstance(error_ref, Ref): 474 raise self.dml.dag.get_error(error_ref) 475 ref = description.get("result") 476 if not isinstance(ref, Ref): 477 raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet") 478 return _make_node(self, ref)
Get the result node of the dag
490 def put(self, value: Any, *, name=None) -> "Node": 491 """ 492 Add a value to the DAG. 493 494 Parameters 495 ---------- 496 value : Union[Scalar, Collection] 497 Value to add 498 name : str, optional 499 Name for the node 500 Returns 501 ------- 502 Node 503 Node representing the value 504 505 Examples 506 -------- 507 >>> n1 = dag.put(42, name="answer") 508 >>> n1.value() 509 42 510 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}) 511 >>> n2.value() 512 {'a': 1, 'b': [42, '23']} 513 >>> n3 = dag.put({"a": 1, "b": [n1, "23"]}) 514 >>> n3.value() 515 {'a': 1, 'b': [42, '23']} 516 """ 517 return _make_node(self, self._put_literal(value, name=name))
Add a value to the DAG.
>>> n1 = dag.put(42, name="answer")
>>> n1.value()
42
>>> n2 = dag.put({"a": 1, "b": [n1, "23"]})
>>> n2.value()
{'a': 1, 'b': [42, '23']}
>>> n3 = dag.put({"a": 1, "b": [n1, "23"]})
>>> n3.value()
{'a': 1, 'b': [42, '23']}
519 def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node": 520 """ 521 Import a node from a different (committed) DAG into the current DAG. 522 523 Parameters 524 ---------- 525 dag_name : str 526 Name of the DAG to import from 527 node_name : str, optional 528 Name of the node to import. If None, imports the result node of the DAG. 529 530 Returns 531 ------- 532 Node 533 The loaded node or DAG 534 535 Examples 536 -------- 537 >>> dag = new(dml=dml, name="test", message="test") 538 >>> n1 = dag.put(42, name="answer") 539 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data") 540 >>> dag.commit(n2) 541 >>> dag2 = new(dml=dml, name="test2", message="test2") 542 >>> imported_n2 = dag2.require("test", "data", name="imported_data") 543 >>> imported_n2.value() 544 {'a': 1, 'b': [42, '23']} 545 """ 546 index = self._require_index_ref() 547 if isinstance(dag_name, Dag): 548 if dag_name.ref is None: 549 raise DmlRepoError("Cannot import an uncommitted DAG") 550 dag_ref = dag_name.ref 551 else: 552 commit = self.dml.runtime.describe(index)["parents"][0] 553 dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name) 554 if dag_ref is None: 555 raise DmlRepoError(f"DAG not found: {dag_name}") 556 dag_info = self.dml.dag.describe(dag_ref) 557 node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result") 558 if node_ref is None: 559 raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'") 560 node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name) 561 return _make_node(self, node_ref)
Import a node from a different (committed) DAG into the current DAG.
>>> dag = new(dml=dml, name="test", message="test")
>>> n1 = dag.put(42, name="answer")
>>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data")
>>> dag.commit(n2)
>>> dag2 = new(dml=dml, name="test2", message="test2")
>>> imported_n2 = dag2.require("test", "data", name="imported_data")
>>> imported_n2.value()
{'a': 1, 'b': [42, '23']}
563 def call( 564 self, 565 fn: Any, 566 *args: Any, 567 name: Optional[str] = None, 568 sleep: Optional[callable] = None, 569 timeout: int = -1, 570 ) -> "Node": 571 """ 572 Call a function node with arguments. 573 574 Parameters 575 ---------- 576 fn : Union[Runnable, RunnableNode] 577 Function to call 578 *args : Union[Node, Scalar, Collection] 579 Arguments to pass to the function 580 name : str, optional 581 Name for the result node 582 sleep : callable, optional 583 A nullary function that returns sleep time in milliseconds 584 timeout : int, default=-1 585 Maximum time to wait in milliseconds. If <= 0, wait indefinitely. 586 587 Returns 588 ------- 589 Node 590 Result node 591 592 Raises 593 ------ 594 TimeoutError 595 If the function call exceeds the timeout 596 Error 597 If the function returns an error 598 """ 599 sleep = sleep or BackoffWithJitter() 600 argv_seed = [fn, *args] 601 end = current_time_millis() + timeout 602 while timeout <= 0 or current_time_millis() < end: 603 argv_refs = [self._put_literal(value) for value in argv_seed] 604 resp = self._start_fn(argv_refs, name=name) 605 if resp: 606 return _make_node(self, resp) 607 time.sleep(sleep() / 1000) 608 raise TimeoutError(f"invoking function: {fn}")
Call a function node with arguments.
610 def commit(self, value) -> None: 611 """ 612 Commit a value to the DAG. 613 614 Parameters 615 ---------- 616 value : Union[Node, Error, Any] 617 Value to commit 618 """ 619 # errors are committed as-is, everything else is a node 620 if not isinstance(value, (Error, Node)): 621 value = self.put(value) 622 if isinstance(value, Node): 623 value = value.ref 624 self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name) 625 self.token = None # Clear the working index since it's now committed
Commit a value to the DAG.
627 def freeze(self, message: str | None = None) -> "Dag": 628 """Freeze this uncommitted DAG's runtime index for read-only inspection.""" 629 freeze_message = f"dag: {self.name}" 630 if message: 631 freeze_message = f"{freeze_message}\n{message}" 632 self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message) 633 return self
Freeze this uncommitted DAG's runtime index for read-only inspection.
640 def dump(self) -> dict[str, Any]: 641 """Return this active or frozen runtime's repository objects as JSON-compatible data. 642 643 This does not freeze execution or preserve locks, execution records, or 644 external artifact bytes. ``dml.runtime.load(data)`` restores the original 645 index ID and overwrites an existing copy. 646 """ 647 return self.dml.runtime.dump(self._require_index_ref())
Return this active or frozen runtime's repository objects as JSON-compatible data.
This does not freeze execution or preserve locks, execution records, or
external artifact bytes. dml.runtime.load(data) restores the original
index ID and overwrites an existing copy.
649 @classmethod 650 def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag": 651 """Construct a DAG wrapper from a runtimepack, preserving its index ID. 652 653 Parameters 654 ---------- 655 data : dict 656 Runtimepack returned by ``Dag.dump()``. 657 dml : Dml, optional 658 Destination repository. Defaults to the active default repository. 659 660 Returns 661 ------- 662 Dag 663 The loaded active or frozen runtime. An existing index is overwritten; 664 execution records and locks are neither created nor restored. 665 """ 666 runtime = dml if dml is not None else get_default_dml() 667 return cls(dml=runtime, token=runtime.runtime.load(data))
Construct a DAG wrapper from a runtimepack, preserving its index ID.
Dag.dump().669 def cancel(self, max_retries: int = 3): 670 """Cancel the DAG's execution. 671 672 Parameters 673 ---------- 674 max_retries : int, default=3 675 Maximum cancellation retries after the initial attempt. 676 """ 677 if self.token is None: 678 raise DmlRepoError("Cannot cancel a committed DAG") 679 logger.info(f"Cancelling execution {self.token}") 680 self.dml.runtime.cancel(self.token, max_retries=max_retries) 681 self.token = None # Clear the index ref to indicate it's no longer active 682 raise CancellationError("DAG execution cancelled")
Cancel the DAG's execution.
685class NodeError(Error): 686 """A stored error enriched with the node that produced it.""" 687 688 def __init__(self, error: Error, *, node_ref: Ref, dag: Dag): 689 super().__init__(message=error.message, origin=error.origin, type=error.type, stack=list(error.stack)) 690 self.node_ref = node_ref 691 self.dag = dag 692 693 def context(self) -> Dag: 694 """Return the function DAG that recorded this node's failure.""" 695 dag = self.dag 696 node_ref = self.node_ref 697 while True: 698 info = dag.dml.dag.describe_node(node_ref) 699 if info["type"] == "FnNode": 700 return Dag(dml=dag.dml, ref=info["dag"]) 701 if info["type"] != "ImportNode": 702 raise DmlRepoError(f"Node {node_ref} does not resolve to a failed function call") 703 dag = Dag(dml=dag.dml, ref=info["dag"]) 704 node_ref = info["node"]
A stored error enriched with the node that produced it.
693 def context(self) -> Dag: 694 """Return the function DAG that recorded this node's failure.""" 695 dag = self.dag 696 node_ref = self.node_ref 697 while True: 698 info = dag.dml.dag.describe_node(node_ref) 699 if info["type"] == "FnNode": 700 return Dag(dml=dag.dml, ref=info["dag"]) 701 if info["type"] != "ImportNode": 702 raise DmlRepoError(f"Node {node_ref} does not resolve to a failed function call") 703 dag = Dag(dml=dag.dml, ref=info["dag"]) 704 node_ref = info["node"]
Return the function DAG that recorded this node's failure.
707@dataclass(frozen=True) 708class Node: # noqa: F811 709 """ 710 Representation of a node in a DaggerML DAG. 711 712 Parameters 713 ---------- 714 dag : Dag 715 Parent DAG 716 ref : Ref 717 Node reference 718 """ 719 720 dag: Dag 721 ref: Ref 722 _info: dict = field(default_factory=dict) 723 724 def __repr__(self): 725 ref_id = self.ref if isinstance(self.ref, Error) else self.ref.to 726 return f"{self.__class__.__name__}({ref_id})" 727 728 def __hash__(self): 729 return hash(self.ref) 730 731 def __eq__(self, other): 732 if not isinstance(other, Node): 733 return NotImplemented 734 return self.ref == other.ref 735 736 def context(self, *, root: bool = True) -> Dag: 737 """ 738 Resolve the provenance context (DAG) for this node. 739 740 This follows import/function provenance while treating builtin 741 collection-construction and selection DAGs as transparent. 742 743 Parameters 744 ---------- 745 root : bool, default=True 746 If False, return the nearest sub-DAG in which this value exists as a 747 proper node across a non-builtin import/function boundary. If True, 748 continue recursively until provenance no longer crosses a 749 non-builtin import/function boundary and return that first rooted 750 context. 751 752 Returns 753 ------- 754 Dag 755 The nearest or rooted provenance DAG for this node. 756 757 Examples 758 -------- 759 >>> source = new(dml=dml, name="source", message="source") 760 >>> answer = source.put(42, name="answer") 761 >>> payload = source.put({"answer": answer}, name="payload") 762 >>> source.commit(payload) 763 >>> consumer = new(dml=dml, name="consumer", message="consumer") 764 >>> imported = consumer.require("source", "payload", name="payload") 765 >>> consumer.commit(imported) 766 >>> loaded = load("consumer", dml=dml) 767 >>> loaded.result["answer"].context() == source 768 True 769 """ 770 return _resolve_context(self, (), root=root) 771 772 @property 773 def type(self): 774 """Get the data type of the node.""" 775 return self._info["data_type"] 776 777 @overload 778 def value(self: "ScalarNode") -> Scalar: ... 779 @overload 780 def value(self: "ListNode") -> list: ... 781 @overload 782 def value(self: "DictNode") -> dict: ... 783 @overload 784 def value(self: "RunnableNode") -> Runnable: ... 785 @overload 786 def value(self: "Node") -> Any: ... 787 def value(self): 788 """ 789 Get the concrete value of this node. 790 791 Returns 792 ------- 793 Any 794 The actual value represented by this node 795 """ 796 value = self.dag.dml.dag.get_node(self.ref, recursive=True) 797 if isinstance(value, Error): 798 raise NodeError(value, node_ref=self.ref, dag=self.dag) 799 return value 800 801 def __call__(self, *args, name=None, sleep=None, timeout=-1, **kw) -> "Node": 802 raise TypeError(f"Node of type '{self.type}' is not callable")
Representation of a node in a DaggerML DAG.
736 def context(self, *, root: bool = True) -> Dag: 737 """ 738 Resolve the provenance context (DAG) for this node. 739 740 This follows import/function provenance while treating builtin 741 collection-construction and selection DAGs as transparent. 742 743 Parameters 744 ---------- 745 root : bool, default=True 746 If False, return the nearest sub-DAG in which this value exists as a 747 proper node across a non-builtin import/function boundary. If True, 748 continue recursively until provenance no longer crosses a 749 non-builtin import/function boundary and return that first rooted 750 context. 751 752 Returns 753 ------- 754 Dag 755 The nearest or rooted provenance DAG for this node. 756 757 Examples 758 -------- 759 >>> source = new(dml=dml, name="source", message="source") 760 >>> answer = source.put(42, name="answer") 761 >>> payload = source.put({"answer": answer}, name="payload") 762 >>> source.commit(payload) 763 >>> consumer = new(dml=dml, name="consumer", message="consumer") 764 >>> imported = consumer.require("source", "payload", name="payload") 765 >>> consumer.commit(imported) 766 >>> loaded = load("consumer", dml=dml) 767 >>> loaded.result["answer"].context() == source 768 True 769 """ 770 return _resolve_context(self, (), root=root)
Resolve the provenance context (DAG) for this node.
This follows import/function provenance while treating builtin collection-construction and selection DAGs as transparent.
>>> source = new(dml=dml, name="source", message="source")
>>> answer = source.put(42, name="answer")
>>> payload = source.put({"answer": answer}, name="payload")
>>> source.commit(payload)
>>> consumer = new(dml=dml, name="consumer", message="consumer")
>>> imported = consumer.require("source", "payload", name="payload")
>>> consumer.commit(imported)
>>> loaded = load("consumer", dml=dml)
>>> loaded.result["answer"].context() == source
True
787 def value(self): 788 """ 789 Get the concrete value of this node. 790 791 Returns 792 ------- 793 Any 794 The actual value represented by this node 795 """ 796 value = self.dag.dml.dag.get_node(self.ref, recursive=True) 797 if isinstance(value, Error): 798 raise NodeError(value, node_ref=self.ref, dag=self.dag) 799 return value
Get the concrete value of this node.
Representation of a node in a DaggerML DAG.
809class RunnableNode(Node): 810 def __call__(self, *args, name=None, sleep=None, timeout=-1) -> "Node": 811 """ 812 Call this node as a function. 813 814 Parameters 815 ---------- 816 *args : Any 817 Arguments to pass to the function 818 name : str, optional 819 Name for the result node 820 sleep : callable, optional 821 A nullary function that returns sleep time in milliseconds 822 timeout : int, default=-1 823 Maximum time to wait in milliseconds. -1 means wait forever. 824 825 Returns 826 ------- 827 Node 828 Result node 829 830 Raises 831 ------ 832 TimeoutError 833 If the function call exceeds the timeout 834 Error 835 If the function returns an error 836 """ 837 return self.dag.call(self, *args, name=name, sleep=sleep, timeout=timeout)
Representation of a node in a DaggerML DAG.
840@dataclass(frozen=True) 841class Projection: 842 dag: Dag 843 base: Node 844 path: tuple[ProjectionStep, ...] 845 _info: dict = field(default_factory=dict) 846 847 def __repr__(self): 848 return f"Projection({self.base!r}, path={self.path!r})" 849 850 @classmethod 851 def from_step(cls, base: Node, step: ProjectionStep) -> "Projection": 852 return cls(dag=base.dag, base=base, path=(step,)) 853 854 def _extend(self, step: ProjectionStep) -> "Projection": 855 return Projection(dag=self.dag, base=self.base, path=(*self.path, step)) 856 857 def value(self): 858 return _apply_projection_path(self.base.value(), self.path) 859 860 def context(self, *, root: bool = True) -> Dag: 861 return _resolve_context(self.base, self.path, root=root) 862 863 def __call__(self, *args, **kwargs): 864 raise TypeError(f"Projection of type '{self.type}' is not callable") 865 866 @property 867 def type(self): 868 if "data_type" in self._info: 869 return self._info["data_type"] 870 return _info_for_value(self.value())["data_type"] 871 872 def __len__(self): 873 if "length" in self._info: 874 return self._info["length"] 875 value = self.value() 876 if not isinstance(value, (list, dict)): 877 raise TypeError(f"Object of type '{type(value).__name__}' has no len()") 878 return len(value) 879 880 def __iter__(self): 881 value = self.value() 882 if isinstance(value, list): 883 for i in range(len(value)): 884 yield self[i] 885 return 886 if isinstance(value, dict): 887 yield from self.keys() 888 return 889 raise TypeError(f"Object of type '{type(value).__name__}' is not iterable") 890 891 def __getitem__(self, key: ProjectionStep | slice) -> "Projection": 892 value = self.value() 893 if isinstance(value, dict): 894 if not isinstance(key, str): 895 raise TypeError(f"Dict keys must be strings but got {type(key).__name__}") 896 step = cast(ProjectionStep, key) 897 elif isinstance(value, list): 898 if not isinstance(key, (int, slice)): 899 raise TypeError(f"List indices must be integers or slices but got {type(key).__name__}") 900 step = _normalize_projection_step(cast(ProjectionStep | slice, key), length=len(value)) 901 else: 902 raise TypeError(f"Cannot project into object of type '{type(value).__name__}'") 903 projected = self._extend(step) 904 return Projection(dag=self.dag, base=self.base, path=projected.path, _info=_info_for_value(projected.value())) 905 906 def keys(self) -> list[str]: 907 value = self.value() 908 if not isinstance(value, dict): 909 raise TypeError(f"Cannot get keys of type: {type(value).__name__}") 910 return sorted(value.keys())
913class CollectionNode(Node): # noqa: F811 914 """ 915 Representation of a collection node in a DaggerML DAG. 916 917 Parameters 918 ---------- 919 dag : Dag 920 Parent DAG 921 ref : Ref 922 Node reference 923 """ 924 925 def contains(self, item, *, name=None) -> "ScalarNode": 926 """ 927 For collection nodes, checks to see if `item` is in `self` 928 929 Returns 930 ------- 931 Node 932 Node with the boolean of is `item` in `self` 933 """ 934 item_ref = item.ref if isinstance(item, Node) else item 935 result = self.dag._call_builtin("daggerml:contains", self.ref, item_ref, name=name) 936 return cast(ScalarNode, _make_node(self.dag, result)) 937 938 def __contains__(self, item): 939 return self.contains(item).value() # has to return boolean 940 941 def __len__(self): # python requires this to be an int 942 """ 943 Get the node's length 944 945 Returns 946 ------- 947 Node 948 Node with the length of the collection 949 950 Raises 951 ------ 952 Error 953 If the node isn't a collection (e.g. list or dict). 954 """ 955 return self._info["length"]
Representation of a collection node in a DaggerML DAG.
925 def contains(self, item, *, name=None) -> "ScalarNode": 926 """ 927 For collection nodes, checks to see if `item` is in `self` 928 929 Returns 930 ------- 931 Node 932 Node with the boolean of is `item` in `self` 933 """ 934 item_ref = item.ref if isinstance(item, Node) else item 935 result = self.dag._call_builtin("daggerml:contains", self.ref, item_ref, name=name) 936 return cast(ScalarNode, _make_node(self.dag, result))
For collection nodes, checks to see if item is in self
item in self958class ListNode(CollectionNode): # noqa: F811 959 """ 960 Representation of a collection node in a DaggerML DAG. 961 962 Parameters 963 ---------- 964 dag : Dag 965 Parent DAG 966 ref : Ref 967 Node reference 968 """ 969 970 @overload 971 def __getitem__(self, key: Union[slice, list[int]]) -> Union["ListNode", Projection]: ... 972 @overload 973 def __getitem__(self, key: Union[int, "Node"]) -> Union["Node", Projection]: ... 974 def __getitem__(self, key: Union[slice, list[int], int, "Node"]) -> Union["Node", Projection]: 975 if self.dag.ref is not None: 976 if isinstance(key, Node): 977 raise TypeError("Committed list projections require concrete int or slice keys") 978 step = _normalize_projection_step(key, length=len(self)) 979 return Projection( 980 dag=self.dag, 981 base=self, 982 path=(step,), 983 _info=_info_for_value(_apply_projection_step(self.value(), step)), 984 ) 985 if isinstance(key, slice): 986 if key.step is not None: 987 raise ValueError("Slice step is not supported") 988 start = key.start if key.start is not None else 0 989 stop = key.stop if key.stop is not None else len(self) 990 key = [start, stop] 991 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key)) 992 993 def __iter__(self): 994 """ 995 Iterate over the node's values (items if it's a list, and keys if it's a 996 dict) 997 998 Returns 999 ------- 1000 Node 1001 Result node 1002 1003 Raises 1004 ------ 1005 Error 1006 If the node isn't a collection (e.g. list or dict). 1007 """ 1008 for i in range(len(self)): 1009 yield self[i] 1010 1011 def conj(self, item, *, name=None) -> "ListNode": 1012 """ 1013 For a list node, append an item 1014 1015 Returns 1016 ------- 1017 Node 1018 Node containing the new collection 1019 1020 Notes 1021 ----- 1022 `append` is an alias `conj` 1023 """ 1024 item_ref = item.ref if isinstance(item, Node) else item 1025 resp = self.dag._call_builtin("daggerml:conj", self.ref, item_ref, name=name) 1026 return cast(ListNode, _make_node(self.dag, resp)) 1027 1028 def append(self, item, *, name=None) -> "ListNode": 1029 """ 1030 For a list node, append an item 1031 1032 Returns 1033 ------- 1034 Node 1035 Node containing the new collection 1036 1037 See Also 1038 -------- 1039 conj : The main implementation 1040 """ 1041 return self.conj(item, name=name)
Representation of a collection node in a DaggerML DAG.
1011 def conj(self, item, *, name=None) -> "ListNode": 1012 """ 1013 For a list node, append an item 1014 1015 Returns 1016 ------- 1017 Node 1018 Node containing the new collection 1019 1020 Notes 1021 ----- 1022 `append` is an alias `conj` 1023 """ 1024 item_ref = item.ref if isinstance(item, Node) else item 1025 resp = self.dag._call_builtin("daggerml:conj", self.ref, item_ref, name=name) 1026 return cast(ListNode, _make_node(self.dag, resp))
1028 def append(self, item, *, name=None) -> "ListNode": 1029 """ 1030 For a list node, append an item 1031 1032 Returns 1033 ------- 1034 Node 1035 Node containing the new collection 1036 1037 See Also 1038 -------- 1039 conj : The main implementation 1040 """ 1041 return self.conj(item, name=name)
For a list node, append an item
conj: The main implementation
1044class DictNode(CollectionNode): # noqa: F811 1045 def __getitem__(self, key: Union[str, "Node"]) -> Union["Node", Projection]: 1046 if self.dag.ref is not None: 1047 if not isinstance(key, str): 1048 raise TypeError(f"Dict keys must be strings but got {type(key).__name__}") 1049 return Projection(dag=self.dag, base=self, path=(key,), _info=_info_for_value(self.value()[key])) 1050 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key)) 1051 1052 def keys(self) -> list[str]: 1053 """ 1054 Get the keys of a dictionary node. 1055 1056 Parameters 1057 ---------- 1058 name : str, optional 1059 Name for the result node 1060 1061 Returns 1062 ------- 1063 list[str] 1064 List of keys in the dictionary node 1065 """ 1066 return self._info["keys"].copy() 1067 1068 def __iter__(self): 1069 """ 1070 Iterate over the node's values (items if it's a list, and keys if it's a 1071 dict) 1072 1073 Returns 1074 ------- 1075 Node 1076 Result node 1077 1078 Raises 1079 ------ 1080 Error 1081 If the node isn't a collection (e.g. list or dict). 1082 """ 1083 for k in self.keys(): 1084 yield k 1085 1086 def get(self, key, default=None, *, name=None) -> "Node": 1087 """ 1088 For a dict node, return the value for key if key exists, else default. 1089 1090 If default is not given, it defaults to None, so that this method never raises a KeyError. 1091 """ 1092 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key, default, name=name)) 1093 1094 def items(self) -> Iterator[tuple[str, "Node|Projection"]]: 1095 """ 1096 Iterate over key-value pairs of a dictionary node. 1097 1098 Returns 1099 ------- 1100 Iterator[tuple[Node, Node]] 1101 Iterator over (key, value) pairs 1102 """ 1103 if self.type != "dict": 1104 raise Error(f"Cannot iterate items of type: {self.type}", origin="dml", type="TypeError") 1105 for k in self: 1106 yield k, self[k] 1107 1108 def values(self) -> list["Node|Projection"]: 1109 """ 1110 Get the values of a dictionary node. 1111 1112 Parameters 1113 ---------- 1114 name : str, optional 1115 Name for the result node 1116 1117 Returns 1118 ------- 1119 list[Node] 1120 List of values in the dictionary node 1121 """ 1122 return [self[k] for k in self] 1123 1124 def assoc(self, key, value, *, name=None) -> "DictNode": 1125 """ 1126 For a dict node, associate a new value into the map 1127 1128 Returns 1129 ------- 1130 Node 1131 Node containing the new dict 1132 """ 1133 value_ref = value.ref if isinstance(value, Node) else value 1134 resp = self.dag._call_builtin("daggerml:assoc", self.ref, key, value_ref, name=name) 1135 return cast(DictNode, _make_node(self.dag, resp)) 1136 1137 def update(self, update) -> "DictNode": 1138 """ 1139 For a dict node, update like python dicts 1140 1141 Returns 1142 ------- 1143 Node 1144 Node containing the new collection 1145 1146 Notes 1147 ----- 1148 calls `assoc` iteratively for k, v pairs in update. 1149 1150 See Also 1151 -------- 1152 assoc : The main implementation 1153 """ 1154 for k, v in update.items(): 1155 self = self.assoc(k, v) 1156 return self
Representation of a collection node in a DaggerML DAG.
1052 def keys(self) -> list[str]: 1053 """ 1054 Get the keys of a dictionary node. 1055 1056 Parameters 1057 ---------- 1058 name : str, optional 1059 Name for the result node 1060 1061 Returns 1062 ------- 1063 list[str] 1064 List of keys in the dictionary node 1065 """ 1066 return self._info["keys"].copy()
Get the keys of a dictionary node.
1086 def get(self, key, default=None, *, name=None) -> "Node": 1087 """ 1088 For a dict node, return the value for key if key exists, else default. 1089 1090 If default is not given, it defaults to None, so that this method never raises a KeyError. 1091 """ 1092 return _make_node(self.dag, self.dag._call_builtin("daggerml:get", self.ref, key, default, name=name))
For a dict node, return the value for key if key exists, else default.
If default is not given, it defaults to None, so that this method never raises a KeyError.
1094 def items(self) -> Iterator[tuple[str, "Node|Projection"]]: 1095 """ 1096 Iterate over key-value pairs of a dictionary node. 1097 1098 Returns 1099 ------- 1100 Iterator[tuple[Node, Node]] 1101 Iterator over (key, value) pairs 1102 """ 1103 if self.type != "dict": 1104 raise Error(f"Cannot iterate items of type: {self.type}", origin="dml", type="TypeError") 1105 for k in self: 1106 yield k, self[k]
Iterate over key-value pairs of a dictionary node.
1108 def values(self) -> list["Node|Projection"]: 1109 """ 1110 Get the values of a dictionary node. 1111 1112 Parameters 1113 ---------- 1114 name : str, optional 1115 Name for the result node 1116 1117 Returns 1118 ------- 1119 list[Node] 1120 List of values in the dictionary node 1121 """ 1122 return [self[k] for k in self]
Get the values of a dictionary node.
1124 def assoc(self, key, value, *, name=None) -> "DictNode": 1125 """ 1126 For a dict node, associate a new value into the map 1127 1128 Returns 1129 ------- 1130 Node 1131 Node containing the new dict 1132 """ 1133 value_ref = value.ref if isinstance(value, Node) else value 1134 resp = self.dag._call_builtin("daggerml:assoc", self.ref, key, value_ref, name=name) 1135 return cast(DictNode, _make_node(self.dag, resp))
For a dict node, associate a new value into the map
1137 def update(self, update) -> "DictNode": 1138 """ 1139 For a dict node, update like python dicts 1140 1141 Returns 1142 ------- 1143 Node 1144 Node containing the new collection 1145 1146 Notes 1147 ----- 1148 calls `assoc` iteratively for k, v pairs in update. 1149 1150 See Also 1151 -------- 1152 assoc : The main implementation 1153 """ 1154 for k, v in update.items(): 1155 self = self.assoc(k, v) 1156 return self
Base class for protocol classes.
Protocol classes are defined as::
class Proto(Protocol):
def meth(self) -> int:
...
Such classes are primarily used with static type checkers that recognize structural subtyping (static duck-typing).
For example::
class C:
def meth(self) -> int:
return 0
def func(x: Proto) -> int:
return x.meth()
func(C()) # Passes static type check
See PEP 544 for details. Protocol classes decorated with @typing.runtime_checkable act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as::
class GenProto[T](Protocol):
def meth(self) -> T:
...
1968def _no_init_or_replace_init(self, *args, **kwargs): 1969 cls = type(self) 1970 1971 if cls._is_protocol: 1972 raise TypeError('Protocols cannot be instantiated') 1973 1974 # Already using a custom `__init__`. No need to calculate correct 1975 # `__init__` to call. This can lead to RecursionError. See bpo-45121. 1976 if cls.__init__ is not _no_init_or_replace_init: 1977 return 1978 1979 # Initially, `__init__` of a protocol subclass is set to `_no_init_or_replace_init`. 1980 # The first instantiation of the subclass will call `_no_init_or_replace_init` which 1981 # searches for a proper new `__init__` in the MRO. The new `__init__` 1982 # replaces the subclass' old `__init__` (ie `_no_init_or_replace_init`). Subsequent 1983 # instantiation of the protocol subclass will thus use the new 1984 # `__init__` and no longer call `_no_init_or_replace_init`. 1985 for base in cls.__mro__: 1986 init = base.__dict__.get('__init__', _no_init_or_replace_init) 1987 if init is not _no_init_or_replace_init: 1988 cls.__init__ = init 1989 break 1990 else: 1991 # should not happen 1992 cls.__init__ = object.__init__ 1993 1994 cls.__init__(self, *args, **kwargs)
1170 def can_encode(self, value: Any) -> bool: ...
Error information with stack traces.
Represents a captured error from a computation, storing error details and stack trace information for debugging.
1187def ensure_literal_codec_plugins_loaded() -> None: 1188 global _plugins_loaded 1189 if _plugins_loaded: 1190 return 1191 with _lock: 1192 codec_seq = 0 1193 if _plugins_loaded: 1194 return 1195 loaded = [] 1196 for entry_point in _entry_points(): 1197 try: 1198 registrations = entry_point.load()() 1199 for item in registrations: 1200 priority, codec = item 1201 codec_seq += 1 1202 loaded.append((priority, codec_seq, codec)) 1203 except Exception as e: 1204 msg = f"Literal codec plugin '{entry_point.name} ({entry_point.value})' failed: {e}" 1205 raise CodecError(msg) from None 1206 loaded.sort(key=lambda item: (-item[0], item[1])) 1207 _codecs.extend(loaded) 1208 _plugins_loaded = True
1216def apply_codec(value: Any, *, dag: Dag) -> Any: 1217 for codec in iter_codecs(): 1218 try: 1219 if codec.can_encode(value): 1220 resp = codec.encode(value, dag) 1221 if isinstance(resp, type(value)): 1222 codec_name = codec.__class__.__name__ 1223 msg = f"Literal codec {codec_name} encoded {value.__class__.__name__} to {resp.__class__.__name__}." 1224 raise CodecError(msg) 1225 return resp 1226 except Exception as e: 1227 if isinstance(e, DmlRepoError): 1228 raise 1229 raise CodecError(f"Literal codec {codec.__class__.__name__} failed: {e}") from e 1230 raise CodecError(f"No codec found for value of type {type(value).__name__}")
1233def apply_codecs(value: Any, *, dag: Dag) -> Any: 1234 while not isinstance(value, (*get_args(Scalar), *get_args(Collection), Ref)): 1235 value = apply_codec(value, dag=dag) 1236 if isinstance(value, list): 1237 return [apply_codecs(v, dag=dag) for v in value] 1238 if isinstance(value, dict): 1239 return {k: apply_codecs(v, dag=dag) for k, v in value.items()} 1240 if isinstance(value, Uri): 1241 return Uri(apply_codecs(value.uri, dag=dag)) 1242 if isinstance(value, Runnable): 1243 target = apply_codecs(value.target, dag=dag) 1244 sub = apply_codecs(value.sub, dag=dag) 1245 kwargs = {k: apply_codecs(v, dag=dag) for k, v in value.kwargs.items()} 1246 return Runnable(target=target, adapter=value.adapter, kwargs=kwargs, sub=sub) 1247 return value
1250class MiscPyTypeCodec: 1251 def can_encode(self, value: Any) -> bool: 1252 return isinstance(value, Mapping) or ( 1253 isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) 1254 ) 1255 1256 def encode(self, value: Sequence | Mapping, dag: Dag) -> Any: 1257 if isinstance(value, Mapping): 1258 return {k: apply_codecs(v, dag=dag) for k, v in value.items()} 1259 if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): 1260 return [apply_codecs(v, dag=dag) for v in value]
1256 def encode(self, value: Sequence | Mapping, dag: Dag) -> Any: 1257 if isinstance(value, Mapping): 1258 return {k: apply_codecs(v, dag=dag) for k, v in value.items()} 1259 if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): 1260 return [apply_codecs(v, dag=dag) for v in value]
1263class NodeCodec: 1264 def can_encode(self, value: Any) -> bool: 1265 return isinstance(value, Node) 1266 1267 def encode(self, value: "Node", dag: Dag) -> Ref: 1268 assert dag.token is not None, "DAG must have a token to encode nodes" 1269 if value.dag.token is not None and value.dag.token == dag.token: 1270 return value.ref 1271 if value.dag.ref is None: 1272 raise CodecError("Cannot encode node from uncommitted DAG in a different index") 1273 try: 1274 return dag.dml.runtime.put_import(dag._require_index_ref(), value.dag.ref, node=value.ref, name=None) 1275 except Exception as e: 1276 raise CodecError(f"Failed to encode cross-dag node import: {e}") from e
1267 def encode(self, value: "Node", dag: Dag) -> Ref: 1268 assert dag.token is not None, "DAG must have a token to encode nodes" 1269 if value.dag.token is not None and value.dag.token == dag.token: 1270 return value.ref 1271 if value.dag.ref is None: 1272 raise CodecError("Cannot encode node from uncommitted DAG in a different index") 1273 try: 1274 return dag.dml.runtime.put_import(dag._require_index_ref(), value.dag.ref, node=value.ref, name=None) 1275 except Exception as e: 1276 raise CodecError(f"Failed to encode cross-dag node import: {e}") from e
1279class ProjectionCodec: 1280 def can_encode(self, value: Any) -> bool: 1281 return isinstance(value, Projection) 1282 1283 def encode(self, value: "Projection", dag: Dag) -> Ref: 1284 assert dag.token is not None, "DAG must have a token to encode projections" 1285 node_ref = NodeCodec().encode(value.base, dag) 1286 for step in value.path: 1287 node_ref = dag._call_builtin("daggerml:get", node_ref, step) 1288 return node_ref
1283 def encode(self, value: "Projection", dag: Dag) -> Ref: 1284 assert dag.token is not None, "DAG must have a token to encode projections" 1285 node_ref = NodeCodec().encode(value.base, dag) 1286 for step in value.path: 1287 node_ref = dag._call_builtin("daggerml:get", node_ref, step) 1288 return node_ref