DaggerML public package exports.
1"""DaggerML public package exports.""" 2 3from daggerml._core import ( 4 BadExecutionStatusError, 5 CanceledExecutionError, 6 CancellationError, 7 Dml, 8 Error, 9 ExecutionDriver, 10 ExecutionMetadata, 11 ExecutionRecord, 12 ExecutionSemanticState, 13 Ref, 14 Runnable, 15 Uri, 16) 17from daggerml.api import ( 18 Dag, 19 Node, 20 clear_default_dml, 21 get_default_dml, 22 load, 23 new, 24 resume, 25 set_default_dml, 26 status, 27 temporary, 28 use_default_dml, 29) 30 31try: 32 from daggerml.__about__ import __version__ 33except ImportError: 34 __version__ = "local" 35 36__all__ = ( 37 "BadExecutionStatusError", 38 "CanceledExecutionError", 39 "CancellationError", 40 "Dag", 41 "Dml", 42 "Error", 43 "ExecutionDriver", 44 "ExecutionMetadata", 45 "ExecutionRecord", 46 "ExecutionSemanticState", 47 "Node", 48 "Ref", 49 "Uri", 50 "Runnable", 51 "get_default_dml", 52 "set_default_dml", 53 "use_default_dml", 54 "clear_default_dml", 55 "new", 56 "load", 57 "resume", 58 "status", 59 "temporary", 60)
484class BadExecutionStatusError(DmlRepoError): 485 """Raised when an execution lifecycle cannot satisfy a requested mutation mode.""" 486 487 def __init__(self, message: str, *, lifecycle: str | None = None): 488 super().__init__(message, type="badexecutionstatuserror") 489 self.lifecycle = lifecycle
Raised when an execution lifecycle cannot satisfy a requested mutation mode.
492class CanceledExecutionError(BadExecutionStatusError): 493 """Raised when cancellation lifecycle blocks activation or mutation.""" 494 495 def __init__(self, message: str, *, lifecycle: str | None = None): 496 super().__init__(message, lifecycle=lifecycle) 497 self.type = "canceledexecutionerror"
Raised when cancellation lifecycle blocks activation or mutation.
Raised when cancellation lifecycle blocks activation or mutation.
340@dataclass 341class Dag: 342 dml: Dml 343 token: Optional[Ref] = None # Working index id 344 ref: Optional[Ref] = None 345 name: str = "" # DAG name for commit 346 message: str = "" # Commit message 347 348 def __repr__(self): 349 to = self.ref.to if self.ref else (self.token.to if self.token is not None else "NA") 350 return f"Dag({to})" 351 352 def __hash__(self): 353 "Useful only for tests." 354 return 42 355 356 def __eq__(self, other): 357 "DAG equality is based on identity, not content." 358 if not isinstance(other, Dag): 359 return False 360 return self.ref == other.ref and self.token == other.token and self.dml == other.dml 361 362 def __enter__(self): 363 "Catch exceptions and commit an Error" 364 assert not self.ref 365 return self 366 367 def __exit__(self, exc_type, exc_value, tb): 368 if exc_value is not None: 369 # Convert exception to Error and commit it 370 traceback.print_exception(exc_type, exc_value, tb) 371 err = Error.from_ex(exc_value) if not isinstance(exc_value, Error) else exc_value 372 self.commit(err) 373 374 def _require_index_ref(self) -> Ref: 375 if self.token is None: 376 raise DmlRepoError("No active index") 377 return self.token 378 379 def _read_dag_ref(self) -> Ref: 380 """Return the completed or partial DAG backing this wrapper's reads.""" 381 if self.ref is not None: 382 return self.ref 383 return cast(Ref, self.dml.runtime.describe(self._require_index_ref())["dag"]) 384 385 @property 386 def tags(self) -> list[str]: 387 """Return the normalized tags stored on this DAG.""" 388 return list(self.dml.dag.describe(self._read_dag_ref())["tags"]) 389 390 def _put_literal(self, value: Any, *, name: Optional[str] = None) -> Ref: 391 index_id = self._require_index_ref() 392 value = apply_codecs(value, dag=self) 393 return self.dml.runtime.put_literal(index_id, value, name=name) 394 395 def _start_fn(self, argv: list[Ref], *, name: Optional[str] = None) -> Optional[Ref]: 396 return self.dml.runtime.start_fn(self._require_index_ref(), argv, name=name) 397 398 def _call_builtin(self, uri: str, *args: Any, name: Optional[str] = None) -> Ref: 399 fn_ref = self._put_literal(Runnable(target=Uri(uri), kwargs={}, adapter="")) 400 argv: list[Ref] = [fn_ref] 401 for arg in args: 402 argv.append(arg if isinstance(arg, Ref) else self._put_literal(arg)) 403 result = self._start_fn(argv, name=name) 404 if result is None: 405 raise DmlRepoError("Function execution failed") 406 return result 407 408 def __len__(self) -> int: 409 return len(self.keys()) 410 411 def __iter__(self): 412 yield from self.keys() 413 414 def _get_named_node(self, name: str) -> "Node": 415 node_ref = self.dml.dag.describe(self._read_dag_ref())["names"].get(name) 416 if node_ref is None: 417 raise DmlRepoError(f"Node '{name}' not found in DAG") 418 return _make_node(self, node_ref) 419 420 def _set_named_node(self, name: str, value: Any) -> None: 421 if self.ref is not None: 422 raise DmlRepoError("Cannot set node names on a committed DAG.") 423 if isinstance(value, Node): 424 value = value.ref 425 if isinstance(value, Ref): 426 self.dml.runtime.set_node_name(self._require_index_ref(), name, value) 427 return 428 self.put(value, name=name) 429 430 def __getitem__(self, name: str) -> "Node": 431 return self._get_named_node(name) 432 433 def __setitem__(self, name: str, value: Any) -> None: 434 self.put(value, name=name) 435 436 def __getattr__(self, name: str) -> "Node": 437 if name.startswith("_"): 438 raise AttributeError(name) 439 return self[name] 440 441 def __setattr__(self, name: str, value: Any) -> None: 442 dataclass_fields = getattr(type(self), "__dataclass_fields__", {}) 443 if name in dataclass_fields or hasattr(type(self), name): 444 object.__setattr__(self, name, value) 445 return 446 self[name] = value 447 448 def keys(self) -> list[str]: 449 """Get the list of all node names in the dag""" 450 names_dict = self.dml.dag.describe(self._read_dag_ref())["names"] 451 return sorted(names_dict.keys()) 452 453 def values(self) -> list["Node"]: 454 """Get the list of all nodes in the dag""" 455 names_dict = self.dml.dag.describe(self._read_dag_ref())["names"] 456 return [_make_node(self, ref) for ref in names_dict.values()] 457 458 @property 459 def argv(self) -> "ListNode": 460 "Access the dag's argv node" 461 argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"] 462 if not isinstance(argv_ref, Ref): 463 raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv") 464 return cast(ListNode, _make_node(self, argv_ref)) 465 466 @property 467 def result(self) -> "Node": 468 """Get the result node of the dag""" 469 if self.ref is None: 470 raise DmlRepoError("Cannot access result of an uncommitted DAG") 471 description = self.dml.dag.describe(self.ref) 472 error_ref = description.get("error") 473 if isinstance(error_ref, Ref): 474 raise self.dml.dag.get_error(error_ref) 475 ref = description.get("result") 476 if not isinstance(ref, Ref): 477 raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet") 478 return _make_node(self, ref) 479 480 @overload 481 def put(self, value: Union[list, "ListNode"], *, name=None) -> "ListNode": ... 482 @overload 483 def put(self, value: Union[dict, "DictNode"], *, name=None) -> "DictNode": ... 484 @overload 485 def put(self, value: Union[Runnable, "RunnableNode"], *, name=None) -> "RunnableNode": ... 486 @overload 487 def put(self, value: Union[Scalar, "ScalarNode"], *, name=None) -> "ScalarNode": ... 488 @overload 489 def put(self, value: Any, *, name=None) -> "Node": ... 490 def put(self, value: Any, *, name=None) -> "Node": 491 """ 492 Add a value to the DAG. 493 494 Parameters 495 ---------- 496 value : Union[Scalar, Collection] 497 Value to add 498 name : str, optional 499 Name for the node 500 Returns 501 ------- 502 Node 503 Node representing the value 504 505 Examples 506 -------- 507 >>> n1 = dag.put(42, name="answer") 508 >>> n1.value() 509 42 510 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}) 511 >>> n2.value() 512 {'a': 1, 'b': [42, '23']} 513 >>> n3 = dag.put({"a": 1, "b": [n1, "23"]}) 514 >>> n3.value() 515 {'a': 1, 'b': [42, '23']} 516 """ 517 return _make_node(self, self._put_literal(value, name=name)) 518 519 def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node": 520 """ 521 Import a node from a different (committed) DAG into the current DAG. 522 523 Parameters 524 ---------- 525 dag_name : str 526 Name of the DAG to import from 527 node_name : str, optional 528 Name of the node to import. If None, imports the result node of the DAG. 529 530 Returns 531 ------- 532 Node 533 The loaded node or DAG 534 535 Examples 536 -------- 537 >>> dag = new(dml=dml, name="test", message="test") 538 >>> n1 = dag.put(42, name="answer") 539 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data") 540 >>> dag.commit(n2) 541 >>> dag2 = new(dml=dml, name="test2", message="test2") 542 >>> imported_n2 = dag2.require("test", "data", name="imported_data") 543 >>> imported_n2.value() 544 {'a': 1, 'b': [42, '23']} 545 """ 546 index = self._require_index_ref() 547 if isinstance(dag_name, Dag): 548 if dag_name.ref is None: 549 raise DmlRepoError("Cannot import an uncommitted DAG") 550 dag_ref = dag_name.ref 551 else: 552 commit = self.dml.runtime.describe(index)["parents"][0] 553 dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name) 554 if dag_ref is None: 555 raise DmlRepoError(f"DAG not found: {dag_name}") 556 dag_info = self.dml.dag.describe(dag_ref) 557 node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result") 558 if node_ref is None: 559 raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'") 560 node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name) 561 return _make_node(self, node_ref) 562 563 def call( 564 self, 565 fn: Any, 566 *args: Any, 567 name: Optional[str] = None, 568 sleep: Optional[callable] = None, 569 timeout: int = -1, 570 ) -> "Node": 571 """ 572 Call a function node with arguments. 573 574 Parameters 575 ---------- 576 fn : Union[Runnable, RunnableNode] 577 Function to call 578 *args : Union[Node, Scalar, Collection] 579 Arguments to pass to the function 580 name : str, optional 581 Name for the result node 582 sleep : callable, optional 583 A nullary function that returns sleep time in milliseconds 584 timeout : int, default=-1 585 Maximum time to wait in milliseconds. If <= 0, wait indefinitely. 586 587 Returns 588 ------- 589 Node 590 Result node 591 592 Raises 593 ------ 594 TimeoutError 595 If the function call exceeds the timeout 596 Error 597 If the function returns an error 598 """ 599 sleep = sleep or BackoffWithJitter() 600 argv_seed = [fn, *args] 601 end = current_time_millis() + timeout 602 while timeout <= 0 or current_time_millis() < end: 603 argv_refs = [self._put_literal(value) for value in argv_seed] 604 resp = self._start_fn(argv_refs, name=name) 605 if resp: 606 return _make_node(self, resp) 607 time.sleep(sleep() / 1000) 608 raise TimeoutError(f"invoking function: {fn}") 609 610 def commit(self, value) -> None: 611 """ 612 Commit a value to the DAG. 613 614 Parameters 615 ---------- 616 value : Union[Node, Error, Any] 617 Value to commit 618 """ 619 # errors are committed as-is, everything else is a node 620 if not isinstance(value, (Error, Node)): 621 value = self.put(value) 622 if isinstance(value, Node): 623 value = value.ref 624 self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name) 625 self.token = None # Clear the working index since it's now committed 626 627 def freeze(self, message: str | None = None) -> "Dag": 628 """Freeze this uncommitted DAG's runtime index for read-only inspection.""" 629 freeze_message = f"dag: {self.name}" 630 if message: 631 freeze_message = f"{freeze_message}\n{message}" 632 self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message) 633 return self 634 635 def unfreeze(self) -> "Dag": 636 """Unfreeze this uncommitted DAG's runtime index.""" 637 self.token = self.dml.runtime.unfreeze(self._require_index_ref()) 638 return self 639 640 def dump(self) -> dict[str, Any]: 641 """Return this active or frozen runtime's repository objects as JSON-compatible data. 642 643 This does not freeze execution or preserve locks, execution records, or 644 external artifact bytes. ``dml.runtime.load(data)`` restores the original 645 index ID and overwrites an existing copy. 646 """ 647 return self.dml.runtime.dump(self._require_index_ref()) 648 649 @classmethod 650 def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag": 651 """Construct a DAG wrapper from a runtimepack, preserving its index ID. 652 653 Parameters 654 ---------- 655 data : dict 656 Runtimepack returned by ``Dag.dump()``. 657 dml : Dml, optional 658 Destination repository. Defaults to the active default repository. 659 660 Returns 661 ------- 662 Dag 663 The loaded active or frozen runtime. An existing index is overwritten; 664 execution records and locks are neither created nor restored. 665 """ 666 runtime = dml if dml is not None else get_default_dml() 667 return cls(dml=runtime, token=runtime.runtime.load(data)) 668 669 def cancel(self, max_retries: int = 3): 670 """Cancel the DAG's execution. 671 672 Parameters 673 ---------- 674 max_retries : int, default=3 675 Maximum cancellation retries after the initial attempt. 676 """ 677 if self.token is None: 678 raise DmlRepoError("Cannot cancel a committed DAG") 679 logger.info(f"Cancelling execution {self.token}") 680 self.dml.runtime.cancel(self.token, max_retries=max_retries) 681 self.token = None # Clear the index ref to indicate it's no longer active 682 raise CancellationError("DAG execution cancelled")
458 @property 459 def argv(self) -> "ListNode": 460 "Access the dag's argv node" 461 argv_ref = self.dml.dag.describe(self._read_dag_ref())["argv"] 462 if not isinstance(argv_ref, Ref): 463 raise DmlRepoError(f"'{self.__class__.__name__}' dag has no argv") 464 return cast(ListNode, _make_node(self, argv_ref))
Access the dag's argv node
466 @property 467 def result(self) -> "Node": 468 """Get the result node of the dag""" 469 if self.ref is None: 470 raise DmlRepoError("Cannot access result of an uncommitted DAG") 471 description = self.dml.dag.describe(self.ref) 472 error_ref = description.get("error") 473 if isinstance(error_ref, Ref): 474 raise self.dml.dag.get_error(error_ref) 475 ref = description.get("result") 476 if not isinstance(ref, Ref): 477 raise DmlRepoError(f"'{self.__class__.__name__}' dag has not been committed yet") 478 return _make_node(self, ref)
Get the result node of the dag
490 def put(self, value: Any, *, name=None) -> "Node": 491 """ 492 Add a value to the DAG. 493 494 Parameters 495 ---------- 496 value : Union[Scalar, Collection] 497 Value to add 498 name : str, optional 499 Name for the node 500 Returns 501 ------- 502 Node 503 Node representing the value 504 505 Examples 506 -------- 507 >>> n1 = dag.put(42, name="answer") 508 >>> n1.value() 509 42 510 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}) 511 >>> n2.value() 512 {'a': 1, 'b': [42, '23']} 513 >>> n3 = dag.put({"a": 1, "b": [n1, "23"]}) 514 >>> n3.value() 515 {'a': 1, 'b': [42, '23']} 516 """ 517 return _make_node(self, self._put_literal(value, name=name))
Add a value to the DAG.
>>> n1 = dag.put(42, name="answer")
>>> n1.value()
42
>>> n2 = dag.put({"a": 1, "b": [n1, "23"]})
>>> n2.value()
{'a': 1, 'b': [42, '23']}
>>> n3 = dag.put({"a": 1, "b": [n1, "23"]})
>>> n3.value()
{'a': 1, 'b': [42, '23']}
519 def require(self, dag_name: str | "Dag", node_name: str | None = None, *, name: str | None = None) -> "Node": 520 """ 521 Import a node from a different (committed) DAG into the current DAG. 522 523 Parameters 524 ---------- 525 dag_name : str 526 Name of the DAG to import from 527 node_name : str, optional 528 Name of the node to import. If None, imports the result node of the DAG. 529 530 Returns 531 ------- 532 Node 533 The loaded node or DAG 534 535 Examples 536 -------- 537 >>> dag = new(dml=dml, name="test", message="test") 538 >>> n1 = dag.put(42, name="answer") 539 >>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data") 540 >>> dag.commit(n2) 541 >>> dag2 = new(dml=dml, name="test2", message="test2") 542 >>> imported_n2 = dag2.require("test", "data", name="imported_data") 543 >>> imported_n2.value() 544 {'a': 1, 'b': [42, '23']} 545 """ 546 index = self._require_index_ref() 547 if isinstance(dag_name, Dag): 548 if dag_name.ref is None: 549 raise DmlRepoError("Cannot import an uncommitted DAG") 550 dag_ref = dag_name.ref 551 else: 552 commit = self.dml.runtime.describe(index)["parents"][0] 553 dag_ref = self.dml.show(revision=commit)["dags"].get(dag_name) 554 if dag_ref is None: 555 raise DmlRepoError(f"DAG not found: {dag_name}") 556 dag_info = self.dml.dag.describe(dag_ref) 557 node_ref = dag_info["names"].get(node_name) if node_name else dag_info.get("result") 558 if node_ref is None: 559 raise DmlRepoError(f"Node '{node_name}' not found in DAG '{dag_name}'") 560 node_ref = self.dml.runtime.put_import(index, dag_ref, node_ref, name=name) 561 return _make_node(self, node_ref)
Import a node from a different (committed) DAG into the current DAG.
>>> dag = new(dml=dml, name="test", message="test")
>>> n1 = dag.put(42, name="answer")
>>> n2 = dag.put({"a": 1, "b": [n1, "23"]}, name="data")
>>> dag.commit(n2)
>>> dag2 = new(dml=dml, name="test2", message="test2")
>>> imported_n2 = dag2.require("test", "data", name="imported_data")
>>> imported_n2.value()
{'a': 1, 'b': [42, '23']}
563 def call( 564 self, 565 fn: Any, 566 *args: Any, 567 name: Optional[str] = None, 568 sleep: Optional[callable] = None, 569 timeout: int = -1, 570 ) -> "Node": 571 """ 572 Call a function node with arguments. 573 574 Parameters 575 ---------- 576 fn : Union[Runnable, RunnableNode] 577 Function to call 578 *args : Union[Node, Scalar, Collection] 579 Arguments to pass to the function 580 name : str, optional 581 Name for the result node 582 sleep : callable, optional 583 A nullary function that returns sleep time in milliseconds 584 timeout : int, default=-1 585 Maximum time to wait in milliseconds. If <= 0, wait indefinitely. 586 587 Returns 588 ------- 589 Node 590 Result node 591 592 Raises 593 ------ 594 TimeoutError 595 If the function call exceeds the timeout 596 Error 597 If the function returns an error 598 """ 599 sleep = sleep or BackoffWithJitter() 600 argv_seed = [fn, *args] 601 end = current_time_millis() + timeout 602 while timeout <= 0 or current_time_millis() < end: 603 argv_refs = [self._put_literal(value) for value in argv_seed] 604 resp = self._start_fn(argv_refs, name=name) 605 if resp: 606 return _make_node(self, resp) 607 time.sleep(sleep() / 1000) 608 raise TimeoutError(f"invoking function: {fn}")
Call a function node with arguments.
610 def commit(self, value) -> None: 611 """ 612 Commit a value to the DAG. 613 614 Parameters 615 ---------- 616 value : Union[Node, Error, Any] 617 Value to commit 618 """ 619 # errors are committed as-is, everything else is a node 620 if not isinstance(value, (Error, Node)): 621 value = self.put(value) 622 if isinstance(value, Node): 623 value = value.ref 624 self.ref = self.dml.runtime.commit(self._require_index_ref(), value, message=self.message, name=self.name) 625 self.token = None # Clear the working index since it's now committed
Commit a value to the DAG.
627 def freeze(self, message: str | None = None) -> "Dag": 628 """Freeze this uncommitted DAG's runtime index for read-only inspection.""" 629 freeze_message = f"dag: {self.name}" 630 if message: 631 freeze_message = f"{freeze_message}\n{message}" 632 self.token = self.dml.runtime.freeze(self._require_index_ref(), message=freeze_message) 633 return self
Freeze this uncommitted DAG's runtime index for read-only inspection.
640 def dump(self) -> dict[str, Any]: 641 """Return this active or frozen runtime's repository objects as JSON-compatible data. 642 643 This does not freeze execution or preserve locks, execution records, or 644 external artifact bytes. ``dml.runtime.load(data)`` restores the original 645 index ID and overwrites an existing copy. 646 """ 647 return self.dml.runtime.dump(self._require_index_ref())
Return this active or frozen runtime's repository objects as JSON-compatible data.
This does not freeze execution or preserve locks, execution records, or
external artifact bytes. dml.runtime.load(data) restores the original
index ID and overwrites an existing copy.
649 @classmethod 650 def load(cls, data: dict[str, Any], *, dml: Dml | None = None) -> "Dag": 651 """Construct a DAG wrapper from a runtimepack, preserving its index ID. 652 653 Parameters 654 ---------- 655 data : dict 656 Runtimepack returned by ``Dag.dump()``. 657 dml : Dml, optional 658 Destination repository. Defaults to the active default repository. 659 660 Returns 661 ------- 662 Dag 663 The loaded active or frozen runtime. An existing index is overwritten; 664 execution records and locks are neither created nor restored. 665 """ 666 runtime = dml if dml is not None else get_default_dml() 667 return cls(dml=runtime, token=runtime.runtime.load(data))
Construct a DAG wrapper from a runtimepack, preserving its index ID.
Dag.dump().669 def cancel(self, max_retries: int = 3): 670 """Cancel the DAG's execution. 671 672 Parameters 673 ---------- 674 max_retries : int, default=3 675 Maximum cancellation retries after the initial attempt. 676 """ 677 if self.token is None: 678 raise DmlRepoError("Cannot cancel a committed DAG") 679 logger.info(f"Cancelling execution {self.token}") 680 self.dml.runtime.cancel(self.token, max_retries=max_retries) 681 self.token = None # Clear the index ref to indicate it's no longer active 682 raise CancellationError("DAG execution cancelled")
Cancel the DAG's execution.
1076class Dml: 1077 def _init_from_config_vars(self, explicit_config: Mapping[str, object]) -> None: 1078 self._explicit_config = dict(explicit_config) 1079 self._config = Config.resolve(explicit=self._explicit_config) 1080 dflt = self._config.default 1081 self._db = DmlDB(self._config.db_path, dflt.db_map_size_headroom, dflt.db_map_size_max) 1082 self._s3_client = None 1083 1084 def __init__( 1085 self, 1086 project_home: Annotated[str | None, "Project root containing the .dml repository."] = None, 1087 *, 1088 db_path: Annotated[str | None, "Override path to the LMDB database."] = None, 1089 db_map_size_headroom: Annotated[int | None, "Extra LMDB map size headroom in bytes."] = None, 1090 db_map_size_max: Annotated[int | None, "Maximum LMDB map size in bytes."] = None, 1091 default_branch_name: Annotated[str | None, "Default branch name for attached HEAD operations."] = None, 1092 remote_root: Annotated[str | None, "Remote storage root URI."] = None, 1093 remote_prune_age_seconds: Annotated[int | None, "Remote GC prune age in seconds."] = None, 1094 remote_fetch_workers: Annotated[int | None, "Number of concurrent remote fetch workers."] = None, 1095 user: Annotated[str | None, "User name recorded in commits and runtime actions."] = None, 1096 config_home: Annotated[str | None, "Override config directory path."] = None, 1097 ): 1098 """Create a DaggerML session bound to one repository and config context.""" 1099 self._init_from_config_vars( 1100 _python_config_vars_to_canonical( 1101 project_home=project_home, 1102 db_path=db_path, 1103 db_map_size_headroom=db_map_size_headroom, 1104 db_map_size_max=db_map_size_max, 1105 default_branch_name=default_branch_name, 1106 remote_root=remote_root, 1107 remote_prune_age_seconds=remote_prune_age_seconds, 1108 remote_fetch_workers=remote_fetch_workers, 1109 user=user, 1110 config_home=config_home, 1111 ) 1112 ) 1113 1114 @classmethod 1115 def from_config_vars( 1116 cls, 1117 config_vars: Annotated[dict[str, object], "Flattened canonical config-var mapping."] | None = None, 1118 ) -> "Dml": 1119 """Create a DaggerML session from flattened canonical config vars.""" 1120 dml = cls.__new__(cls) 1121 dml._init_from_config_vars(config_vars or {}) 1122 return dml 1123 1124 @classmethod 1125 def init( 1126 cls, 1127 project_home: Annotated[str, "Directory where the repository should be initialized."] = ".", 1128 *, 1129 db_path: Annotated[str | None, "Override path to the LMDB database."] = None, 1130 db_map_size_headroom: Annotated[int | None, "Extra LMDB map size headroom in bytes."] = None, 1131 db_map_size_max: Annotated[int | None, "Maximum LMDB map size in bytes."] = None, 1132 default_branch_name: Annotated[str | None, "Default branch name for attached HEAD operations."] = None, 1133 remote_root: Annotated[str | None, "Remote storage root URI."] = None, 1134 remote_prune_age_seconds: Annotated[int | None, "Remote GC prune age in seconds."] = None, 1135 remote_fetch_workers: Annotated[int | None, "Number of concurrent remote fetch workers."] = None, 1136 user: Annotated[str | None, "User name recorded in commits and runtime actions."] = None, 1137 config_home: Annotated[str | None, "Override config directory path."] = None, 1138 branch: Annotated[str | None, "Initial branch name."] = None, 1139 ) -> "Dml": 1140 """Initialize a repository with an unborn attached HEAD.""" 1141 config = Config.init(project_home, remote_root=remote_root) 1142 dml = cls.from_config_vars( 1143 _python_config_vars_to_canonical( 1144 project_home=config.project_home, 1145 db_path=db_path, 1146 db_map_size_headroom=db_map_size_headroom, 1147 db_map_size_max=db_map_size_max, 1148 default_branch_name=default_branch_name, 1149 remote_root=remote_root, 1150 remote_prune_age_seconds=remote_prune_age_seconds, 1151 remote_fetch_workers=remote_fetch_workers, 1152 user=user, 1153 config_home=config_home, 1154 ) 1155 ) 1156 head = Head(config.project_home) 1157 branch = branch or dml._config.default.branch_name 1158 with head.lock(): 1159 try: 1160 head.get_head() 1161 except (DmlRepoError, FileNotFoundError): 1162 dml._db.init() 1163 head.init(None, branch) 1164 return dml 1165 1166 @classmethod 1167 def clone( 1168 cls, 1169 revision: Annotated[Ref | str | None, "Optional branch, tag, or commit revision."] = None, 1170 /, 1171 *, 1172 project_home: Annotated[str, "Directory where the repository should be cloned."] = ".", 1173 db_path: Annotated[str | None, "Override path to the LMDB database."] = None, 1174 db_map_size_headroom: Annotated[int | None, "Extra LMDB map size headroom in bytes."] = None, 1175 db_map_size_max: Annotated[int | None, "Maximum LMDB map size in bytes."] = None, 1176 default_branch_name: Annotated[str | None, "Default branch name for attached HEAD operations."] = None, 1177 remote_root: Annotated[str | None, "Remote storage root URI."] = None, 1178 remote_prune_age_seconds: Annotated[int | None, "Remote GC prune age in seconds."] = None, 1179 remote_fetch_workers: Annotated[int | None, "Number of concurrent remote fetch workers."] = None, 1180 user: Annotated[str | None, "User name recorded in commits and runtime actions."] = None, 1181 config_home: Annotated[str | None, "Override config directory path."] = None, 1182 depth: Annotated[int | None, "Positive number of commit-history generations to fetch."] = None, 1183 ) -> "Dml": 1184 """Clone one revision from the configured remote root.""" 1185 _validate_history_options(depth) 1186 Path(project_home).mkdir(parents=True, exist_ok=True) 1187 config = Config.init(project_home, remote_root=remote_root) 1188 dml = cls.from_config_vars( 1189 _python_config_vars_to_canonical( 1190 project_home=config.project_home, 1191 db_path=db_path, 1192 db_map_size_headroom=db_map_size_headroom, 1193 db_map_size_max=db_map_size_max, 1194 default_branch_name=default_branch_name, 1195 remote_root=remote_root, 1196 remote_prune_age_seconds=remote_prune_age_seconds, 1197 remote_fetch_workers=remote_fetch_workers, 1198 user=user, 1199 config_home=config_home, 1200 ) 1201 ) 1202 _require_remote_root(dml) 1203 selected = revision or dml._config.default.branch_name 1204 exact = selected if isinstance(selected, Ref) else None 1205 if isinstance(selected, str) and re.match(r"^(?:commit:)?[0-9a-f]{64}$", selected): 1206 exact = Ref(selected if selected.startswith("commit:") else f"commit:{selected}") 1207 branch = ( 1208 selected 1209 if exact is None and isinstance(selected, str) and not selected.startswith(("@", "HEAD")) 1210 else None 1211 ) 1212 initial_branch = branch or dml._config.default.branch_name 1213 head = Head(config.project_home) 1214 with head.lock(): 1215 try: 1216 head.get_head() 1217 except (DmlRepoError, FileNotFoundError): 1218 dml._db.init() 1219 head.init(None, initial_branch) 1220 else: 1221 raise DmlRepoError(f"Cannot clone into an initialized repository: {dml._config.project_home}") 1222 if exact is not None: 1223 commit, available, omitted = _remote_ops(dml).materialize_project_commit_ref(exact, dml._db, depth=depth) 1224 with head.lock(): 1225 _publish_shallow_state(head, available, omitted) 1226 head.write_detached_head(commit) 1227 return dml 1228 dml.fetch(selected if isinstance(selected, str) else None, depth=depth) 1229 kind = "tag" if isinstance(selected, str) and selected.startswith("@") else "branch" 1230 name = selected[1:] if kind == "tag" and isinstance(selected, str) else selected 1231 assert isinstance(name, str) 1232 with head.lock(): 1233 commit = head.get_remote_tracking_ref(name, kind=kind) 1234 if kind == "tag": 1235 head.write_detached_head(commit) 1236 else: 1237 head.update_local_ref(name, commit, kind="branch") 1238 head.write_attached_head(name) 1239 head.set_upstream(name, name) 1240 return dml 1241 1242 def status(self) -> StatusPayload: 1243 """Return branch, commit, and open-runtime status for this repository.""" 1244 head = _head_ops(self) 1245 head_info = head.get_head() 1246 ahead = behind = None 1247 upstream = head.get_upstream(head_info["branch"]) if head_info["branch"] is not None else None 1248 if upstream is not None: 1249 try: 1250 upstream_ref = head.get_remote_tracking_ref(upstream["branch"]) 1251 except DmlRepoError: 1252 pass 1253 else: 1254 if head_info["commit"] is not None: 1255 try: 1256 ahead, behind = CommitOps().ahead_behind( 1257 head_info["commit"], 1258 upstream_ref, 1259 db=self._db, 1260 missing_commits=head.get_shallow_commits(), 1261 ) 1262 except ShallowHistoryError: 1263 pass 1264 with self._db.tx(readonly=True) as txn: 1265 num_indexes = 0 1266 for namespace in ("index", "frozenindex"): 1267 try: 1268 num_indexes += sum(1 for _ in txn.iter(namespace)) 1269 except DmlDbKeyNotFoundError: 1270 pass 1271 return { 1272 "mode": head_info["mode"], 1273 "branch": head_info["branch"], 1274 "commit": head_info["commit"], 1275 "branches": head.list_local_refs(kind="branch"), 1276 "upstream": upstream["branch"] if upstream is not None else None, 1277 "num_indexes": num_indexes, 1278 "ahead": ahead, 1279 "behind": behind, 1280 } 1281 1282 def log( 1283 self, 1284 revision: Annotated[Ref | str, "Revision to start the log from."] = "HEAD", 1285 limit: Annotated[int, "Maximum number of commits to return."] = 10, 1286 *, 1287 remote: Annotated[bool, "Resolve from fetched remote tracking refs."] = False, 1288 dep: Annotated[str | None, "Resolve from a fetched dependency."] = None, 1289 ) -> LogPayload: 1290 """Return commit history starting from one revision.""" 1291 commit_ref = resolve_rev(_head_ops(self), revision, db=self._db, remote=remote, dep=dep) 1292 commit_ref = _require_resolved_commit(commit_ref, revision) 1293 commits, truncated = CommitOps().log_with_truncation( 1294 commit_ref, 1295 limit=limit, 1296 db=self._db, 1297 missing_commits=_head_ops(self).get_shallow_commits(), 1298 ) 1299 return {"commits": commits, "truncated": truncated} 1300 1301 def show( 1302 self, 1303 revision: Annotated[Ref | str, "Revision to describe."] = "HEAD", 1304 *, 1305 remote: bool = False, 1306 dep: str | None = None, 1307 ) -> CommitFullDescription: 1308 """Return a full commit description for one revision.""" 1309 commit_ref = resolve_rev(_head_ops(self), revision, db=self._db, remote=remote, dep=dep) 1310 commit_ref = _require_resolved_commit(commit_ref, revision) 1311 return CommitOps().show( 1312 commit_ref, 1313 db=self._db, 1314 missing_commits=_head_ops(self).get_shallow_commits(), 1315 ) 1316 1317 def diff( 1318 self, 1319 revision: Annotated[Ref | str, "Revision to diff."] = "HEAD", 1320 relative_to: Annotated[Ref | str | None, "Optional base revision. Defaults to the commit parent."] = None, 1321 *, 1322 remote: Annotated[bool, "Resolve the primary revision from remote tracking."] = False, 1323 dep: Annotated[str | None, "Resolve the primary revision from a dependency."] = None, 1324 ) -> CommitDiffPayload: 1325 """Return DAG-level changes for one revision.""" 1326 head = _head_ops(self) 1327 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote, dep=dep) 1328 commit_ref = _require_resolved_commit(commit_ref, revision) 1329 if relative_to is None: 1330 return CommitOps().diff( 1331 commit_ref, 1332 db=self._db, 1333 missing_commits=head.get_shallow_commits(), 1334 ) 1335 rel_to_commit = resolve_rev(head, relative_to, db=self._db) 1336 rel_to_commit = _require_resolved_commit(rel_to_commit, relative_to) 1337 return CommitOps().diff( 1338 commit_ref, 1339 rel_to_commit, 1340 db=self._db, 1341 missing_commits=head.get_shallow_commits(), 1342 ) 1343 1344 def rev_parse( 1345 self, 1346 revision: Annotated[str, "Revision expression to resolve."], 1347 *, 1348 remote: bool = False, 1349 dep: str | None = None, 1350 ) -> RevisionPayload: 1351 """Resolve a revision expression into a commit and ref metadata.""" 1352 head = _head_ops(self) 1353 branch = tag = None 1354 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote, dep=dep) 1355 if revision.startswith("HEAD"): 1356 kind = "head" 1357 elif re.match(r"^(?:commit:)?[0-9a-f]{64}$", revision): 1358 kind = "commit" 1359 else: 1360 kind = "ref" 1361 if kind == "ref": 1362 branch, tag = (None, revision[1:]) if revision.startswith("@") else (revision, None) 1363 return { 1364 "input": revision, 1365 "uri": None, 1366 "kind": kind, 1367 "commit": commit_ref, 1368 "branch": branch, 1369 "tag": tag, 1370 } 1371 1372 def revert( 1373 self, 1374 revision: Annotated[Ref | str, "Revision whose changes should be reverted."], 1375 message: Annotated[str | None, "Optional commit message for the revert commit."] = None, 1376 *, 1377 remote: bool = False, 1378 ) -> StatusPayload: 1379 """Revert the changes introduced by one revision.""" 1380 head = _head_ops(self) 1381 with head.lock(): 1382 head_info = head.get_head() 1383 if head_info["branch"] is None: 1384 raise DmlRepoError("Cannot revert when HEAD is detached") 1385 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1386 commit_ref = _require_resolved_commit(commit_ref, revision) 1387 new_commit = CommitOps().revert( 1388 commit_ref, 1389 _require_resolved_commit(head_info["commit"], "HEAD"), 1390 user=self._config.user, 1391 message=message, 1392 db=self._db, 1393 missing_commits=head.get_shallow_commits(), 1394 ) 1395 head.update_local_ref(head_info["branch"], new_commit, kind="branch") 1396 return self.status() 1397 1398 def checkout( 1399 self, revision: Annotated[Ref | str, "Revision to check out."], *, remote: bool = False 1400 ) -> StatusPayload: 1401 """Check out a different revision. 1402 1403 If the revision resolves to a local branch, HEAD stays attached to that branch. 1404 Other revisions detach HEAD at the resolved commit. 1405 """ 1406 head = _head_ops(self) 1407 with head.lock(): 1408 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1409 if ( 1410 not remote 1411 and isinstance(revision, str) 1412 and not revision.startswith(("@", "HEAD")) 1413 and not re.match(r"^(?:commit:)?[0-9a-f]{64}$", revision) 1414 ): 1415 head.write_attached_head(revision) 1416 else: 1417 head.write_detached_head(_require_resolved_commit(commit_ref, revision)) 1418 return self.status() 1419 1420 def merge( 1421 self, 1422 revision: Annotated[Ref | str, "Revision to merge into the current branch."], 1423 ff_only: Annotated[bool, "Whether to only allow fast-forward merges."] = True, 1424 *, 1425 remote: bool = False, 1426 ) -> StatusPayload: 1427 """Merge a revision into the current HEAD.""" 1428 head = _head_ops(self) 1429 with head.lock(): 1430 head_info = head.get_head() 1431 if head_info["branch"] is None: 1432 raise DmlRepoError("Cannot merge when HEAD is detached") 1433 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1434 commit_ref = _require_resolved_commit(commit_ref, revision) 1435 new_commit = CommitOps().merge( 1436 head_info["commit"], 1437 commit_ref, 1438 user=self._config.user, 1439 ff_only=ff_only, 1440 db=self._db, 1441 missing_commits=head.get_shallow_commits(), 1442 ) 1443 head.update_local_ref(head_info["branch"], new_commit) 1444 return self.status() 1445 1446 def rebase( 1447 self, revision: Annotated[Ref | str, "Revision to rebase the current branch onto."], *, remote: bool = False 1448 ) -> StatusPayload: 1449 """Rebase the current HEAD onto a different revision.""" 1450 head = _head_ops(self) 1451 with head.lock(): 1452 head_info = head.get_head() 1453 if head_info["branch"] is None: 1454 raise DmlRepoError("Cannot rebase when HEAD is detached") 1455 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1456 new_commit = CommitOps().rebase( 1457 _require_resolved_commit(head_info["commit"], "HEAD"), 1458 _require_resolved_commit(commit_ref, revision), 1459 user=self._config.user, 1460 db=self._db, 1461 missing_commits=head.get_shallow_commits(), 1462 ) 1463 head.update_local_ref(head_info["branch"], new_commit) 1464 return self.status() 1465 1466 def fetch( 1467 self, 1468 revision: Annotated[str | None, "Branch or @tag to fetch."] = None, 1469 /, 1470 *, 1471 dep: Annotated[str | None, "Named dependency endpoint to fetch from."] = None, 1472 depth: Annotated[int | None, "Positive number of commit-history generations to fetch."] = None, 1473 unshallow: Annotated[bool, "Fetch all history through existing shallow boundaries."] = False, 1474 ) -> None: 1475 """Fetch one branch or tag from remote.root or a named dependency.""" 1476 _validate_history_options(depth, unshallow) 1477 selector = revision or self._config.default.branch_name 1478 kind = "tag" if selector.startswith("@") else "branch" 1479 name = selector[1:] if kind == "tag" else selector 1480 head = _head_ops(self) 1481 if dep is None: 1482 remote_ops = _remote_ops(self) 1483 else: 1484 config = head.get_dependency_config(dep) 1485 remote_ops = Remote( 1486 config["root"], 1487 n_workers=self._config.remote.fetch_workers, 1488 client=_require_s3_client(self), 1489 prune_age_seconds=self._config.remote.prune_age_seconds, 1490 ) 1491 materialized = remote_ops.get_project_commit_ref( 1492 kind, 1493 name, 1494 db=self._db, 1495 depth=depth, 1496 unshallow=unshallow, 1497 ) 1498 if materialized is None: 1499 raise DmlRepoError(f"Remote {kind} ref not found: {selector}") 1500 commit, available, omitted = materialized 1501 with head.lock(): 1502 _publish_shallow_state(head, available, omitted) 1503 if dep is None: 1504 head.update_remote_tracking_ref(name, commit, kind=kind) 1505 else: 1506 head.update_dependency_ref(dep, name, commit, kind=kind) 1507 1508 def pull( 1509 self, 1510 ff_only: Annotated[bool, "Whether to only allow fast-forward merges."] = True, 1511 *, 1512 depth: Annotated[int | None, "Positive number of commit-history generations to fetch."] = None, 1513 ) -> StatusPayload: 1514 """Fetch and merge the current branch's configured upstream.""" 1515 _validate_history_options(depth) 1516 head = _head_ops(self) 1517 head_info = head.get_head() 1518 if head_info["branch"] is None: 1519 raise DmlRepoError("Cannot pull when HEAD is detached") 1520 upstream = head.get_upstream(head_info["branch"]) 1521 if upstream is None: 1522 raise DmlRepoError(f"Cannot pull untracked branch: {head_info['branch']}") 1523 self.fetch(upstream["branch"], depth=depth) 1524 self.merge(upstream["branch"], ff_only=ff_only, remote=True) 1525 return self.status() 1526 1527 def push( 1528 self, 1529 *, 1530 revision: Annotated[Ref | str, "Local revision to publish; defaults to HEAD."] = "HEAD", 1531 force: Annotated[bool, "Overwrite a remote branch or tag without publication checks."] = False, 1532 ) -> None: 1533 """Publish a local revision as a tag or branch on the configured remote.""" 1534 head = _head_ops(self) 1535 commit_ref = _require_resolved_commit(resolve_rev(head, revision, db=self._db), revision) 1536 if isinstance(revision, str) and revision.startswith("@"): 1537 _remote_ops(self).put_ref( 1538 commit_ref, 1539 kind="tag", 1540 name=revision[1:], 1541 db=self._db, 1542 force=force, 1543 missing_commits=head.get_shallow_commits(), 1544 ) 1545 return 1546 named_branch = ( 1547 isinstance(revision, str) 1548 and not revision.startswith("HEAD") 1549 and re.fullmatch(r"(?:commit:)?[0-9a-f]{64}", revision) is None 1550 ) 1551 branch = revision if named_branch and isinstance(revision, str) else head.get_head()["branch"] 1552 if branch is None: 1553 raise DmlRepoError("Cannot push an unnamed revision when HEAD is detached") 1554 upstream = head.get_upstream(branch) 1555 if upstream is None: 1556 upstream = {"branch": branch} 1557 set_upstream = True 1558 else: 1559 set_upstream = False 1560 remote = _remote_ops(self) 1561 remote.put_ref( 1562 commit_ref, 1563 kind="branch", 1564 name=upstream["branch"], 1565 db=self._db, 1566 force=force, 1567 missing_commits=head.get_shallow_commits(), 1568 ) 1569 if set_upstream: 1570 with head.lock(): 1571 head.set_upstream(branch, upstream["branch"]) 1572 1573 def gc( 1574 self, 1575 *, 1576 remote: Annotated[bool, "Garbage-collect configured remote state instead of local objects."] = False, 1577 unrecognized: Annotated[ 1578 Literal["raise", "ignore"], 1579 "Remote only: fail on (raise) or log and skip (ignore) unrecognized objects in immutable storage.", 1580 ] = "raise", 1581 ) -> LocalGCSummary | RemoteGCSummary: 1582 """Garbage-collect unreachable local or configured remote state.""" 1583 if not remote: 1584 return _local_gc(self) 1585 return cast(RemoteGCSummary, _remote_ops(self).gc(unrecognized)) 1586 1587 @property 1588 def branch(self) -> Annotated[_BranchNamespace, "Branch inspection and lifecycle commands."]: 1589 """Expose branch inspection and lifecycle commands.""" 1590 return _BranchNamespace(self) 1591 1592 @property 1593 def dep(self) -> Annotated[_DependencyNamespace, "Import-only dependency lifecycle commands."]: 1594 """Expose import-only dependency lifecycle commands.""" 1595 return _DependencyNamespace(self) 1596 1597 @property 1598 def cache(self) -> Annotated[_CacheNamespace, "Remote execution cache inspection and control commands."]: 1599 """Expose remote execution cache inspection and control commands.""" 1600 return _CacheNamespace(self) 1601 1602 @property 1603 def tag(self) -> Annotated[_TagNamespace, "Tag inspection and lifecycle commands."]: 1604 """Expose tag inspection and lifecycle commands.""" 1605 return _TagNamespace(self) 1606 1607 @property 1608 def config(self) -> Annotated[_ConfigNamespace, "Configuration commands."]: 1609 """Expose configuration commands.""" 1610 return _ConfigNamespace(self) 1611 1612 @property 1613 def runtime(self) -> Annotated[_RuntimeNamespace, "Runtime mutation and execution-state inspection commands."]: 1614 """Expose runtime mutation and execution-state inspection commands.""" 1615 return _RuntimeNamespace(self) 1616 1617 @property 1618 def dag(self) -> Annotated[_DagNamespace, "Committed DAG inspection commands."]: 1619 """Expose committed DAG inspection commands.""" 1620 return _DagNamespace(self) 1621 1622 @property 1623 def skills(self) -> Annotated[_SkillsNamespace, "Bundled agent-guidance exports."]: 1624 """Expose bundled agent-guidance exports.""" 1625 return _SkillsNamespace(self)
1084 def __init__( 1085 self, 1086 project_home: Annotated[str | None, "Project root containing the .dml repository."] = None, 1087 *, 1088 db_path: Annotated[str | None, "Override path to the LMDB database."] = None, 1089 db_map_size_headroom: Annotated[int | None, "Extra LMDB map size headroom in bytes."] = None, 1090 db_map_size_max: Annotated[int | None, "Maximum LMDB map size in bytes."] = None, 1091 default_branch_name: Annotated[str | None, "Default branch name for attached HEAD operations."] = None, 1092 remote_root: Annotated[str | None, "Remote storage root URI."] = None, 1093 remote_prune_age_seconds: Annotated[int | None, "Remote GC prune age in seconds."] = None, 1094 remote_fetch_workers: Annotated[int | None, "Number of concurrent remote fetch workers."] = None, 1095 user: Annotated[str | None, "User name recorded in commits and runtime actions."] = None, 1096 config_home: Annotated[str | None, "Override config directory path."] = None, 1097 ): 1098 """Create a DaggerML session bound to one repository and config context.""" 1099 self._init_from_config_vars( 1100 _python_config_vars_to_canonical( 1101 project_home=project_home, 1102 db_path=db_path, 1103 db_map_size_headroom=db_map_size_headroom, 1104 db_map_size_max=db_map_size_max, 1105 default_branch_name=default_branch_name, 1106 remote_root=remote_root, 1107 remote_prune_age_seconds=remote_prune_age_seconds, 1108 remote_fetch_workers=remote_fetch_workers, 1109 user=user, 1110 config_home=config_home, 1111 ) 1112 )
Create a DaggerML session bound to one repository and config context.
1114 @classmethod 1115 def from_config_vars( 1116 cls, 1117 config_vars: Annotated[dict[str, object], "Flattened canonical config-var mapping."] | None = None, 1118 ) -> "Dml": 1119 """Create a DaggerML session from flattened canonical config vars.""" 1120 dml = cls.__new__(cls) 1121 dml._init_from_config_vars(config_vars or {}) 1122 return dml
Create a DaggerML session from flattened canonical config vars.
1124 @classmethod 1125 def init( 1126 cls, 1127 project_home: Annotated[str, "Directory where the repository should be initialized."] = ".", 1128 *, 1129 db_path: Annotated[str | None, "Override path to the LMDB database."] = None, 1130 db_map_size_headroom: Annotated[int | None, "Extra LMDB map size headroom in bytes."] = None, 1131 db_map_size_max: Annotated[int | None, "Maximum LMDB map size in bytes."] = None, 1132 default_branch_name: Annotated[str | None, "Default branch name for attached HEAD operations."] = None, 1133 remote_root: Annotated[str | None, "Remote storage root URI."] = None, 1134 remote_prune_age_seconds: Annotated[int | None, "Remote GC prune age in seconds."] = None, 1135 remote_fetch_workers: Annotated[int | None, "Number of concurrent remote fetch workers."] = None, 1136 user: Annotated[str | None, "User name recorded in commits and runtime actions."] = None, 1137 config_home: Annotated[str | None, "Override config directory path."] = None, 1138 branch: Annotated[str | None, "Initial branch name."] = None, 1139 ) -> "Dml": 1140 """Initialize a repository with an unborn attached HEAD.""" 1141 config = Config.init(project_home, remote_root=remote_root) 1142 dml = cls.from_config_vars( 1143 _python_config_vars_to_canonical( 1144 project_home=config.project_home, 1145 db_path=db_path, 1146 db_map_size_headroom=db_map_size_headroom, 1147 db_map_size_max=db_map_size_max, 1148 default_branch_name=default_branch_name, 1149 remote_root=remote_root, 1150 remote_prune_age_seconds=remote_prune_age_seconds, 1151 remote_fetch_workers=remote_fetch_workers, 1152 user=user, 1153 config_home=config_home, 1154 ) 1155 ) 1156 head = Head(config.project_home) 1157 branch = branch or dml._config.default.branch_name 1158 with head.lock(): 1159 try: 1160 head.get_head() 1161 except (DmlRepoError, FileNotFoundError): 1162 dml._db.init() 1163 head.init(None, branch) 1164 return dml
Initialize a repository with an unborn attached HEAD.
1166 @classmethod 1167 def clone( 1168 cls, 1169 revision: Annotated[Ref | str | None, "Optional branch, tag, or commit revision."] = None, 1170 /, 1171 *, 1172 project_home: Annotated[str, "Directory where the repository should be cloned."] = ".", 1173 db_path: Annotated[str | None, "Override path to the LMDB database."] = None, 1174 db_map_size_headroom: Annotated[int | None, "Extra LMDB map size headroom in bytes."] = None, 1175 db_map_size_max: Annotated[int | None, "Maximum LMDB map size in bytes."] = None, 1176 default_branch_name: Annotated[str | None, "Default branch name for attached HEAD operations."] = None, 1177 remote_root: Annotated[str | None, "Remote storage root URI."] = None, 1178 remote_prune_age_seconds: Annotated[int | None, "Remote GC prune age in seconds."] = None, 1179 remote_fetch_workers: Annotated[int | None, "Number of concurrent remote fetch workers."] = None, 1180 user: Annotated[str | None, "User name recorded in commits and runtime actions."] = None, 1181 config_home: Annotated[str | None, "Override config directory path."] = None, 1182 depth: Annotated[int | None, "Positive number of commit-history generations to fetch."] = None, 1183 ) -> "Dml": 1184 """Clone one revision from the configured remote root.""" 1185 _validate_history_options(depth) 1186 Path(project_home).mkdir(parents=True, exist_ok=True) 1187 config = Config.init(project_home, remote_root=remote_root) 1188 dml = cls.from_config_vars( 1189 _python_config_vars_to_canonical( 1190 project_home=config.project_home, 1191 db_path=db_path, 1192 db_map_size_headroom=db_map_size_headroom, 1193 db_map_size_max=db_map_size_max, 1194 default_branch_name=default_branch_name, 1195 remote_root=remote_root, 1196 remote_prune_age_seconds=remote_prune_age_seconds, 1197 remote_fetch_workers=remote_fetch_workers, 1198 user=user, 1199 config_home=config_home, 1200 ) 1201 ) 1202 _require_remote_root(dml) 1203 selected = revision or dml._config.default.branch_name 1204 exact = selected if isinstance(selected, Ref) else None 1205 if isinstance(selected, str) and re.match(r"^(?:commit:)?[0-9a-f]{64}$", selected): 1206 exact = Ref(selected if selected.startswith("commit:") else f"commit:{selected}") 1207 branch = ( 1208 selected 1209 if exact is None and isinstance(selected, str) and not selected.startswith(("@", "HEAD")) 1210 else None 1211 ) 1212 initial_branch = branch or dml._config.default.branch_name 1213 head = Head(config.project_home) 1214 with head.lock(): 1215 try: 1216 head.get_head() 1217 except (DmlRepoError, FileNotFoundError): 1218 dml._db.init() 1219 head.init(None, initial_branch) 1220 else: 1221 raise DmlRepoError(f"Cannot clone into an initialized repository: {dml._config.project_home}") 1222 if exact is not None: 1223 commit, available, omitted = _remote_ops(dml).materialize_project_commit_ref(exact, dml._db, depth=depth) 1224 with head.lock(): 1225 _publish_shallow_state(head, available, omitted) 1226 head.write_detached_head(commit) 1227 return dml 1228 dml.fetch(selected if isinstance(selected, str) else None, depth=depth) 1229 kind = "tag" if isinstance(selected, str) and selected.startswith("@") else "branch" 1230 name = selected[1:] if kind == "tag" and isinstance(selected, str) else selected 1231 assert isinstance(name, str) 1232 with head.lock(): 1233 commit = head.get_remote_tracking_ref(name, kind=kind) 1234 if kind == "tag": 1235 head.write_detached_head(commit) 1236 else: 1237 head.update_local_ref(name, commit, kind="branch") 1238 head.write_attached_head(name) 1239 head.set_upstream(name, name) 1240 return dml
Clone one revision from the configured remote root.
1242 def status(self) -> StatusPayload: 1243 """Return branch, commit, and open-runtime status for this repository.""" 1244 head = _head_ops(self) 1245 head_info = head.get_head() 1246 ahead = behind = None 1247 upstream = head.get_upstream(head_info["branch"]) if head_info["branch"] is not None else None 1248 if upstream is not None: 1249 try: 1250 upstream_ref = head.get_remote_tracking_ref(upstream["branch"]) 1251 except DmlRepoError: 1252 pass 1253 else: 1254 if head_info["commit"] is not None: 1255 try: 1256 ahead, behind = CommitOps().ahead_behind( 1257 head_info["commit"], 1258 upstream_ref, 1259 db=self._db, 1260 missing_commits=head.get_shallow_commits(), 1261 ) 1262 except ShallowHistoryError: 1263 pass 1264 with self._db.tx(readonly=True) as txn: 1265 num_indexes = 0 1266 for namespace in ("index", "frozenindex"): 1267 try: 1268 num_indexes += sum(1 for _ in txn.iter(namespace)) 1269 except DmlDbKeyNotFoundError: 1270 pass 1271 return { 1272 "mode": head_info["mode"], 1273 "branch": head_info["branch"], 1274 "commit": head_info["commit"], 1275 "branches": head.list_local_refs(kind="branch"), 1276 "upstream": upstream["branch"] if upstream is not None else None, 1277 "num_indexes": num_indexes, 1278 "ahead": ahead, 1279 "behind": behind, 1280 }
Return branch, commit, and open-runtime status for this repository.
1282 def log( 1283 self, 1284 revision: Annotated[Ref | str, "Revision to start the log from."] = "HEAD", 1285 limit: Annotated[int, "Maximum number of commits to return."] = 10, 1286 *, 1287 remote: Annotated[bool, "Resolve from fetched remote tracking refs."] = False, 1288 dep: Annotated[str | None, "Resolve from a fetched dependency."] = None, 1289 ) -> LogPayload: 1290 """Return commit history starting from one revision.""" 1291 commit_ref = resolve_rev(_head_ops(self), revision, db=self._db, remote=remote, dep=dep) 1292 commit_ref = _require_resolved_commit(commit_ref, revision) 1293 commits, truncated = CommitOps().log_with_truncation( 1294 commit_ref, 1295 limit=limit, 1296 db=self._db, 1297 missing_commits=_head_ops(self).get_shallow_commits(), 1298 ) 1299 return {"commits": commits, "truncated": truncated}
Return commit history starting from one revision.
1301 def show( 1302 self, 1303 revision: Annotated[Ref | str, "Revision to describe."] = "HEAD", 1304 *, 1305 remote: bool = False, 1306 dep: str | None = None, 1307 ) -> CommitFullDescription: 1308 """Return a full commit description for one revision.""" 1309 commit_ref = resolve_rev(_head_ops(self), revision, db=self._db, remote=remote, dep=dep) 1310 commit_ref = _require_resolved_commit(commit_ref, revision) 1311 return CommitOps().show( 1312 commit_ref, 1313 db=self._db, 1314 missing_commits=_head_ops(self).get_shallow_commits(), 1315 )
Return a full commit description for one revision.
1317 def diff( 1318 self, 1319 revision: Annotated[Ref | str, "Revision to diff."] = "HEAD", 1320 relative_to: Annotated[Ref | str | None, "Optional base revision. Defaults to the commit parent."] = None, 1321 *, 1322 remote: Annotated[bool, "Resolve the primary revision from remote tracking."] = False, 1323 dep: Annotated[str | None, "Resolve the primary revision from a dependency."] = None, 1324 ) -> CommitDiffPayload: 1325 """Return DAG-level changes for one revision.""" 1326 head = _head_ops(self) 1327 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote, dep=dep) 1328 commit_ref = _require_resolved_commit(commit_ref, revision) 1329 if relative_to is None: 1330 return CommitOps().diff( 1331 commit_ref, 1332 db=self._db, 1333 missing_commits=head.get_shallow_commits(), 1334 ) 1335 rel_to_commit = resolve_rev(head, relative_to, db=self._db) 1336 rel_to_commit = _require_resolved_commit(rel_to_commit, relative_to) 1337 return CommitOps().diff( 1338 commit_ref, 1339 rel_to_commit, 1340 db=self._db, 1341 missing_commits=head.get_shallow_commits(), 1342 )
Return DAG-level changes for one revision.
1344 def rev_parse( 1345 self, 1346 revision: Annotated[str, "Revision expression to resolve."], 1347 *, 1348 remote: bool = False, 1349 dep: str | None = None, 1350 ) -> RevisionPayload: 1351 """Resolve a revision expression into a commit and ref metadata.""" 1352 head = _head_ops(self) 1353 branch = tag = None 1354 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote, dep=dep) 1355 if revision.startswith("HEAD"): 1356 kind = "head" 1357 elif re.match(r"^(?:commit:)?[0-9a-f]{64}$", revision): 1358 kind = "commit" 1359 else: 1360 kind = "ref" 1361 if kind == "ref": 1362 branch, tag = (None, revision[1:]) if revision.startswith("@") else (revision, None) 1363 return { 1364 "input": revision, 1365 "uri": None, 1366 "kind": kind, 1367 "commit": commit_ref, 1368 "branch": branch, 1369 "tag": tag, 1370 }
Resolve a revision expression into a commit and ref metadata.
1372 def revert( 1373 self, 1374 revision: Annotated[Ref | str, "Revision whose changes should be reverted."], 1375 message: Annotated[str | None, "Optional commit message for the revert commit."] = None, 1376 *, 1377 remote: bool = False, 1378 ) -> StatusPayload: 1379 """Revert the changes introduced by one revision.""" 1380 head = _head_ops(self) 1381 with head.lock(): 1382 head_info = head.get_head() 1383 if head_info["branch"] is None: 1384 raise DmlRepoError("Cannot revert when HEAD is detached") 1385 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1386 commit_ref = _require_resolved_commit(commit_ref, revision) 1387 new_commit = CommitOps().revert( 1388 commit_ref, 1389 _require_resolved_commit(head_info["commit"], "HEAD"), 1390 user=self._config.user, 1391 message=message, 1392 db=self._db, 1393 missing_commits=head.get_shallow_commits(), 1394 ) 1395 head.update_local_ref(head_info["branch"], new_commit, kind="branch") 1396 return self.status()
Revert the changes introduced by one revision.
1398 def checkout( 1399 self, revision: Annotated[Ref | str, "Revision to check out."], *, remote: bool = False 1400 ) -> StatusPayload: 1401 """Check out a different revision. 1402 1403 If the revision resolves to a local branch, HEAD stays attached to that branch. 1404 Other revisions detach HEAD at the resolved commit. 1405 """ 1406 head = _head_ops(self) 1407 with head.lock(): 1408 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1409 if ( 1410 not remote 1411 and isinstance(revision, str) 1412 and not revision.startswith(("@", "HEAD")) 1413 and not re.match(r"^(?:commit:)?[0-9a-f]{64}$", revision) 1414 ): 1415 head.write_attached_head(revision) 1416 else: 1417 head.write_detached_head(_require_resolved_commit(commit_ref, revision)) 1418 return self.status()
Check out a different revision.
If the revision resolves to a local branch, HEAD stays attached to that branch. Other revisions detach HEAD at the resolved commit.
1420 def merge( 1421 self, 1422 revision: Annotated[Ref | str, "Revision to merge into the current branch."], 1423 ff_only: Annotated[bool, "Whether to only allow fast-forward merges."] = True, 1424 *, 1425 remote: bool = False, 1426 ) -> StatusPayload: 1427 """Merge a revision into the current HEAD.""" 1428 head = _head_ops(self) 1429 with head.lock(): 1430 head_info = head.get_head() 1431 if head_info["branch"] is None: 1432 raise DmlRepoError("Cannot merge when HEAD is detached") 1433 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1434 commit_ref = _require_resolved_commit(commit_ref, revision) 1435 new_commit = CommitOps().merge( 1436 head_info["commit"], 1437 commit_ref, 1438 user=self._config.user, 1439 ff_only=ff_only, 1440 db=self._db, 1441 missing_commits=head.get_shallow_commits(), 1442 ) 1443 head.update_local_ref(head_info["branch"], new_commit) 1444 return self.status()
Merge a revision into the current HEAD.
1446 def rebase( 1447 self, revision: Annotated[Ref | str, "Revision to rebase the current branch onto."], *, remote: bool = False 1448 ) -> StatusPayload: 1449 """Rebase the current HEAD onto a different revision.""" 1450 head = _head_ops(self) 1451 with head.lock(): 1452 head_info = head.get_head() 1453 if head_info["branch"] is None: 1454 raise DmlRepoError("Cannot rebase when HEAD is detached") 1455 commit_ref = resolve_rev(head, revision, db=self._db, remote=remote) 1456 new_commit = CommitOps().rebase( 1457 _require_resolved_commit(head_info["commit"], "HEAD"), 1458 _require_resolved_commit(commit_ref, revision), 1459 user=self._config.user, 1460 db=self._db, 1461 missing_commits=head.get_shallow_commits(), 1462 ) 1463 head.update_local_ref(head_info["branch"], new_commit) 1464 return self.status()
Rebase the current HEAD onto a different revision.
1466 def fetch( 1467 self, 1468 revision: Annotated[str | None, "Branch or @tag to fetch."] = None, 1469 /, 1470 *, 1471 dep: Annotated[str | None, "Named dependency endpoint to fetch from."] = None, 1472 depth: Annotated[int | None, "Positive number of commit-history generations to fetch."] = None, 1473 unshallow: Annotated[bool, "Fetch all history through existing shallow boundaries."] = False, 1474 ) -> None: 1475 """Fetch one branch or tag from remote.root or a named dependency.""" 1476 _validate_history_options(depth, unshallow) 1477 selector = revision or self._config.default.branch_name 1478 kind = "tag" if selector.startswith("@") else "branch" 1479 name = selector[1:] if kind == "tag" else selector 1480 head = _head_ops(self) 1481 if dep is None: 1482 remote_ops = _remote_ops(self) 1483 else: 1484 config = head.get_dependency_config(dep) 1485 remote_ops = Remote( 1486 config["root"], 1487 n_workers=self._config.remote.fetch_workers, 1488 client=_require_s3_client(self), 1489 prune_age_seconds=self._config.remote.prune_age_seconds, 1490 ) 1491 materialized = remote_ops.get_project_commit_ref( 1492 kind, 1493 name, 1494 db=self._db, 1495 depth=depth, 1496 unshallow=unshallow, 1497 ) 1498 if materialized is None: 1499 raise DmlRepoError(f"Remote {kind} ref not found: {selector}") 1500 commit, available, omitted = materialized 1501 with head.lock(): 1502 _publish_shallow_state(head, available, omitted) 1503 if dep is None: 1504 head.update_remote_tracking_ref(name, commit, kind=kind) 1505 else: 1506 head.update_dependency_ref(dep, name, commit, kind=kind)
Fetch one branch or tag from remote.root or a named dependency.
1508 def pull( 1509 self, 1510 ff_only: Annotated[bool, "Whether to only allow fast-forward merges."] = True, 1511 *, 1512 depth: Annotated[int | None, "Positive number of commit-history generations to fetch."] = None, 1513 ) -> StatusPayload: 1514 """Fetch and merge the current branch's configured upstream.""" 1515 _validate_history_options(depth) 1516 head = _head_ops(self) 1517 head_info = head.get_head() 1518 if head_info["branch"] is None: 1519 raise DmlRepoError("Cannot pull when HEAD is detached") 1520 upstream = head.get_upstream(head_info["branch"]) 1521 if upstream is None: 1522 raise DmlRepoError(f"Cannot pull untracked branch: {head_info['branch']}") 1523 self.fetch(upstream["branch"], depth=depth) 1524 self.merge(upstream["branch"], ff_only=ff_only, remote=True) 1525 return self.status()
Fetch and merge the current branch's configured upstream.
1527 def push( 1528 self, 1529 *, 1530 revision: Annotated[Ref | str, "Local revision to publish; defaults to HEAD."] = "HEAD", 1531 force: Annotated[bool, "Overwrite a remote branch or tag without publication checks."] = False, 1532 ) -> None: 1533 """Publish a local revision as a tag or branch on the configured remote.""" 1534 head = _head_ops(self) 1535 commit_ref = _require_resolved_commit(resolve_rev(head, revision, db=self._db), revision) 1536 if isinstance(revision, str) and revision.startswith("@"): 1537 _remote_ops(self).put_ref( 1538 commit_ref, 1539 kind="tag", 1540 name=revision[1:], 1541 db=self._db, 1542 force=force, 1543 missing_commits=head.get_shallow_commits(), 1544 ) 1545 return 1546 named_branch = ( 1547 isinstance(revision, str) 1548 and not revision.startswith("HEAD") 1549 and re.fullmatch(r"(?:commit:)?[0-9a-f]{64}", revision) is None 1550 ) 1551 branch = revision if named_branch and isinstance(revision, str) else head.get_head()["branch"] 1552 if branch is None: 1553 raise DmlRepoError("Cannot push an unnamed revision when HEAD is detached") 1554 upstream = head.get_upstream(branch) 1555 if upstream is None: 1556 upstream = {"branch": branch} 1557 set_upstream = True 1558 else: 1559 set_upstream = False 1560 remote = _remote_ops(self) 1561 remote.put_ref( 1562 commit_ref, 1563 kind="branch", 1564 name=upstream["branch"], 1565 db=self._db, 1566 force=force, 1567 missing_commits=head.get_shallow_commits(), 1568 ) 1569 if set_upstream: 1570 with head.lock(): 1571 head.set_upstream(branch, upstream["branch"])
Publish a local revision as a tag or branch on the configured remote.
1573 def gc( 1574 self, 1575 *, 1576 remote: Annotated[bool, "Garbage-collect configured remote state instead of local objects."] = False, 1577 unrecognized: Annotated[ 1578 Literal["raise", "ignore"], 1579 "Remote only: fail on (raise) or log and skip (ignore) unrecognized objects in immutable storage.", 1580 ] = "raise", 1581 ) -> LocalGCSummary | RemoteGCSummary: 1582 """Garbage-collect unreachable local or configured remote state.""" 1583 if not remote: 1584 return _local_gc(self) 1585 return cast(RemoteGCSummary, _remote_ops(self).gc(unrecognized))
Garbage-collect unreachable local or configured remote state.
Expose branch inspection and lifecycle commands.
Expose import-only dependency lifecycle commands.
Expose remote execution cache inspection and control commands.
Expose tag inspection and lifecycle commands.
Expose configuration commands.
Expose runtime mutation and execution-state inspection commands.
Expose committed DAG inspection commands.
395@_register_dml_obj 396class Error(DmlBase, Exception): 397 """Error information with stack traces. 398 399 Represents a captured error from a computation, storing error details 400 and stack trace information for debugging. 401 402 Attributes 403 ---------- 404 message : str 405 The error message. 406 origin : str 407 The origin/source of the error (e.g., 'python', 'adapter'). 408 type : str 409 The error type name. 410 stack : list[dict] 411 Stack trace frames as dictionaries. 412 """ 413 414 message: str 415 origin: str 416 type: str 417 stack: list[dict] = field(default_factory=list) 418 419 def __post_init__(self): 420 """Initialize Exception base with message and run base initialization.""" 421 Exception.__init__(self, self.message) 422 423 def _validate(self) -> None: 424 if not isinstance(self.message, str): 425 raise TypeError(f"{self.__class__.__name__}.message must be a string") 426 if not isinstance(self.origin, str): 427 raise TypeError(f"{self.__class__.__name__}.origin must be a string") 428 if not isinstance(self.type, str): 429 raise TypeError(f"{self.__class__.__name__}.type must be a string") 430 if not isinstance(self.stack, list): 431 raise TypeError(f"{self.__class__.__name__}.stack must be a list of frame dicts") 432 for frame in self.stack: 433 if not isinstance(frame, dict): 434 raise TypeError(f"{self.__class__.__name__}.stack frame must be a dict") 435 436 @classmethod 437 def from_ex(cls, exc) -> "Error": 438 """Create Error from Python exception. 439 440 Parameters 441 ---------- 442 exc : Exception 443 Python exception to convert. 444 445 Returns 446 ------- 447 Error 448 Error object with extracted stack trace. 449 """ 450 if isinstance(exc, Error): 451 return Error(message=exc.message, origin=exc.origin, type=exc.type, stack=list(exc.stack)) 452 tb = traceback.extract_tb(exc.__traceback__) 453 stack = [ 454 { 455 "filename": frame.filename, 456 "lineno": frame.lineno, 457 "name": frame.name, 458 "line": frame.line, 459 } 460 for frame in tb 461 ] 462 return cls( 463 message=str(exc), 464 origin="python", 465 type=type(exc).__name__.lower(), 466 stack=stack, 467 )
Error information with stack traces.
Represents a captured error from a computation, storing error details and stack trace information for debugging.
436 @classmethod 437 def from_ex(cls, exc) -> "Error": 438 """Create Error from Python exception. 439 440 Parameters 441 ---------- 442 exc : Exception 443 Python exception to convert. 444 445 Returns 446 ------- 447 Error 448 Error object with extracted stack trace. 449 """ 450 if isinstance(exc, Error): 451 return Error(message=exc.message, origin=exc.origin, type=exc.type, stack=list(exc.stack)) 452 tb = traceback.extract_tb(exc.__traceback__) 453 stack = [ 454 { 455 "filename": frame.filename, 456 "lineno": frame.lineno, 457 "name": frame.name, 458 "line": frame.line, 459 } 460 for frame in tb 461 ] 462 return cls( 463 message=str(exc), 464 origin="python", 465 type=type(exc).__name__.lower(), 466 stack=stack, 467 )
Create Error from Python exception.
62class ExecutionSemanticState(TypedDict): 63 lifecycle: EXECUTION_LIFECYCLES 64 result_ref: str | None 65 result_source: Literal["runtime", "adapter-error"] | None 66 spawned_execution_ids: list[str] 67 child_execution_ids: list[str] 68 cancelation: ControlRecord | None 69 invalidation: ControlRecord | None 70 updated_at: int
707@dataclass(frozen=True) 708class Node: # noqa: F811 709 """ 710 Representation of a node in a DaggerML DAG. 711 712 Parameters 713 ---------- 714 dag : Dag 715 Parent DAG 716 ref : Ref 717 Node reference 718 """ 719 720 dag: Dag 721 ref: Ref 722 _info: dict = field(default_factory=dict) 723 724 def __repr__(self): 725 ref_id = self.ref if isinstance(self.ref, Error) else self.ref.to 726 return f"{self.__class__.__name__}({ref_id})" 727 728 def __hash__(self): 729 return hash(self.ref) 730 731 def __eq__(self, other): 732 if not isinstance(other, Node): 733 return NotImplemented 734 return self.ref == other.ref 735 736 def context(self, *, root: bool = True) -> Dag: 737 """ 738 Resolve the provenance context (DAG) for this node. 739 740 This follows import/function provenance while treating builtin 741 collection-construction and selection DAGs as transparent. 742 743 Parameters 744 ---------- 745 root : bool, default=True 746 If False, return the nearest sub-DAG in which this value exists as a 747 proper node across a non-builtin import/function boundary. If True, 748 continue recursively until provenance no longer crosses a 749 non-builtin import/function boundary and return that first rooted 750 context. 751 752 Returns 753 ------- 754 Dag 755 The nearest or rooted provenance DAG for this node. 756 757 Examples 758 -------- 759 >>> source = new(dml=dml, name="source", message="source") 760 >>> answer = source.put(42, name="answer") 761 >>> payload = source.put({"answer": answer}, name="payload") 762 >>> source.commit(payload) 763 >>> consumer = new(dml=dml, name="consumer", message="consumer") 764 >>> imported = consumer.require("source", "payload", name="payload") 765 >>> consumer.commit(imported) 766 >>> loaded = load("consumer", dml=dml) 767 >>> loaded.result["answer"].context() == source 768 True 769 """ 770 return _resolve_context(self, (), root=root) 771 772 @property 773 def type(self): 774 """Get the data type of the node.""" 775 return self._info["data_type"] 776 777 @overload 778 def value(self: "ScalarNode") -> Scalar: ... 779 @overload 780 def value(self: "ListNode") -> list: ... 781 @overload 782 def value(self: "DictNode") -> dict: ... 783 @overload 784 def value(self: "RunnableNode") -> Runnable: ... 785 @overload 786 def value(self: "Node") -> Any: ... 787 def value(self): 788 """ 789 Get the concrete value of this node. 790 791 Returns 792 ------- 793 Any 794 The actual value represented by this node 795 """ 796 value = self.dag.dml.dag.get_node(self.ref, recursive=True) 797 if isinstance(value, Error): 798 raise NodeError(value, node_ref=self.ref, dag=self.dag) 799 return value 800 801 def __call__(self, *args, name=None, sleep=None, timeout=-1, **kw) -> "Node": 802 raise TypeError(f"Node of type '{self.type}' is not callable")
Representation of a node in a DaggerML DAG.
736 def context(self, *, root: bool = True) -> Dag: 737 """ 738 Resolve the provenance context (DAG) for this node. 739 740 This follows import/function provenance while treating builtin 741 collection-construction and selection DAGs as transparent. 742 743 Parameters 744 ---------- 745 root : bool, default=True 746 If False, return the nearest sub-DAG in which this value exists as a 747 proper node across a non-builtin import/function boundary. If True, 748 continue recursively until provenance no longer crosses a 749 non-builtin import/function boundary and return that first rooted 750 context. 751 752 Returns 753 ------- 754 Dag 755 The nearest or rooted provenance DAG for this node. 756 757 Examples 758 -------- 759 >>> source = new(dml=dml, name="source", message="source") 760 >>> answer = source.put(42, name="answer") 761 >>> payload = source.put({"answer": answer}, name="payload") 762 >>> source.commit(payload) 763 >>> consumer = new(dml=dml, name="consumer", message="consumer") 764 >>> imported = consumer.require("source", "payload", name="payload") 765 >>> consumer.commit(imported) 766 >>> loaded = load("consumer", dml=dml) 767 >>> loaded.result["answer"].context() == source 768 True 769 """ 770 return _resolve_context(self, (), root=root)
Resolve the provenance context (DAG) for this node.
This follows import/function provenance while treating builtin collection-construction and selection DAGs as transparent.
>>> source = new(dml=dml, name="source", message="source")
>>> answer = source.put(42, name="answer")
>>> payload = source.put({"answer": answer}, name="payload")
>>> source.commit(payload)
>>> consumer = new(dml=dml, name="consumer", message="consumer")
>>> imported = consumer.require("source", "payload", name="payload")
>>> consumer.commit(imported)
>>> loaded = load("consumer", dml=dml)
>>> loaded.result["answer"].context() == source
True
787 def value(self): 788 """ 789 Get the concrete value of this node. 790 791 Returns 792 ------- 793 Any 794 The actual value represented by this node 795 """ 796 value = self.dag.dml.dag.get_node(self.ref, recursive=True) 797 if isinstance(value, Error): 798 raise NodeError(value, node_ref=self.ref, dag=self.dag) 799 return value
Get the concrete value of this node.
Reference to another node.
Ref distinguishes stored references from plain strings, which is needed
so serialization can round-trip graph edges instead of raw text.
Initialize a reference wrapper.
namespace:id form.to is not a string.Return the namespace portion of the reference.
This uses the database C parser so Python and C agree on ref structure.
Return the identifier portion of the reference.
This complements ns() by exposing the ID while keeping the split
logic centralized in the database layer.
319@dataclass 320class Runnable: 321 target: Uri 322 sub: Optional["Runnable"] = None 323 kwargs: dict[str, Any] = field(default_factory=dict) 324 adapter: str = "" 325 326 def innermost(self) -> "Runnable": 327 """Get the innermost Runnable in the chain.""" 328 current = self 329 while current.sub is not None: 330 current = current.sub 331 return current
Return the active default Dml runtime.
Set the process-default Dml runtime.
Temporarily override the default Dml runtime for the active context.
Clear the process-default Dml runtime.
74def new( 75 name="", 76 message="", 77 cache_key: str | None = None, 78 execution_id: str | None = None, 79 tags: list[str] | None = None, 80 dml: Dml | None = None, 81) -> "Dag": 82 """Create a new DAG using the active or provided Dml runtime.""" 83 runtime = dml or get_default_dml() 84 execution = Ref(f"index:{execution_id}") if execution_id is not None else None 85 create_kwargs = {"cache_key": cache_key, "execution": execution} 86 if tags is not None: 87 create_kwargs["tags"] = tags 88 index_id = runtime.runtime.create(**create_kwargs) 89 return Dag(dml=runtime, token=index_id, name=name, message=message)
Create a new DAG using the active or provided Dml runtime.
92def load( 93 name: str, 94 dml: Dml | None = None, 95 *, 96 revision: Ref | str = "HEAD", 97 remote: bool = False, 98 dep: str | None = None, 99) -> "Dag": 100 """Load a DAG using the active default Dml runtime.""" 101 dml = dml or get_default_dml() 102 dag_ref = dml.show(revision, remote=remote, dep=dep)["dags"].get(name) 103 if dag_ref is None: 104 raise DmlRepoError(f"DAG not found: {name}") 105 return Dag(dml=dml, ref=dag_ref, name=name)
Load a DAG using the active default Dml runtime.
108def resume( 109 frozen: Ref, 110 *, 111 name: str, 112 message: str, 113 dml: Dml | None = None, 114) -> "Dag": 115 """Resume a frozen DAG runtime with explicitly supplied commit metadata.""" 116 runtime = dml or get_default_dml() 117 index_id = runtime.runtime.unfreeze(frozen) 118 return Dag(dml=runtime, token=index_id, name=name, message=message)
Resume a frozen DAG runtime with explicitly supplied commit metadata.
128def status() -> dict[str, object]: 129 """Return status for the active default Dml runtime.""" 130 dml, source = _resolve_default_dml(create=True) 131 return { 132 "default": { 133 "source": source, 134 "has_scoped_override": _SCOPED_DEFAULT_DML.get() is not _NO_DEFAULT_DML, 135 "has_process_default": _PROCESS_DEFAULT_DML is not None, 136 }, 137 "status": dml.status(), 138 }
Return status for the active default Dml runtime.
Create a temporary Dml runtime with an unborn attached HEAD.