kiln_ai.datamodel.basemodel
1import json 2import uuid 3from abc import ABCMeta 4from builtins import classmethod 5from datetime import datetime 6from pathlib import Path 7from typing import ( 8 TYPE_CHECKING, 9 Any, 10 Dict, 11 List, 12 Optional, 13 Self, 14 Type, 15 TypeVar, 16) 17 18from pydantic import ( 19 BaseModel, 20 ConfigDict, 21 Field, 22 ValidationError, 23 computed_field, 24 model_validator, 25) 26from pydantic_core import ErrorDetails 27 28from kiln_ai.utils.config import Config 29from kiln_ai.utils.formatting import snake_case 30 31# ID is a 12 digit random integer string. 32# Should be unique per item, at least inside the context of a parent/child relationship. 33# Use integers to make it easier to type for a search function. 34# Allow none, even though we generate it, because we clear it in the REST API if the object is ephemeral (not persisted to disk) 35ID_FIELD = Field(default_factory=lambda: str(uuid.uuid4().int)[:12]) 36ID_TYPE = Optional[str] 37T = TypeVar("T", bound="KilnBaseModel") 38PT = TypeVar("PT", bound="KilnParentedModel") 39 40 41class KilnBaseModel(BaseModel): 42 """Base model for all Kiln data models with common functionality for persistence and versioning. 43 44 Attributes: 45 v (int): Schema version number for migration support 46 id (str): Unique identifier for the model instance 47 path (Path): File system path where the model is stored 48 created_at (datetime): Timestamp when the model was created 49 created_by (str): User ID of the creator 50 """ 51 52 model_config = ConfigDict(validate_assignment=True) 53 54 v: int = Field(default=1) # schema_version 55 id: ID_TYPE = ID_FIELD 56 path: Optional[Path] = Field(default=None) 57 created_at: datetime = Field(default_factory=datetime.now) 58 created_by: str = Field(default_factory=lambda: Config.shared().user_id) 59 60 @computed_field() 61 def model_type(self) -> str: 62 return self.type_name() 63 64 # if changing the model name, should keep the original name here for parsing old files 65 @classmethod 66 def type_name(cls) -> str: 67 return snake_case(cls.__name__) 68 69 # used as /obj_folder/base_filename.kiln 70 @classmethod 71 def base_filename(cls) -> str: 72 return cls.type_name() + ".kiln" 73 74 @classmethod 75 def load_from_folder(cls: Type[T], folderPath: Path) -> T: 76 """Load a model instance from a folder using the default filename. 77 78 Args: 79 folderPath (Path): Directory path containing the model file 80 81 Returns: 82 T: Instance of the model 83 """ 84 path = folderPath / cls.base_filename() 85 return cls.load_from_file(path) 86 87 @classmethod 88 def load_from_file(cls: Type[T], path: Path) -> T: 89 """Load a model instance from a specific file path. 90 91 Args: 92 path (Path): Path to the model file 93 94 Returns: 95 T: Instance of the model 96 97 Raises: 98 ValueError: If the loaded model is not of the expected type or version 99 """ 100 with open(path, "r") as file: 101 file_data = file.read() 102 # TODO P2 perf: parsing the JSON twice here. 103 # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums. 104 parsed_json = json.loads(file_data) 105 m = cls.model_validate_json(file_data, strict=True) 106 if not isinstance(m, cls): 107 raise ValueError(f"Loaded model is not of type {cls.__name__}") 108 file_data = None 109 m.path = path 110 if m.v > m.max_schema_version(): 111 raise ValueError( 112 f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. " 113 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 114 f"version: {m.v}, max version: {m.max_schema_version()}" 115 ) 116 if parsed_json["model_type"] != cls.type_name(): 117 raise ValueError( 118 f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. " 119 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 120 f"version: {m.v}, max version: {m.max_schema_version()}" 121 ) 122 return m 123 124 def save_to_file(self) -> None: 125 """Save the model instance to a file. 126 127 Raises: 128 ValueError: If the path is not set 129 """ 130 path = self.build_path() 131 if path is None: 132 raise ValueError( 133 f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, " 134 f"id: {getattr(self, 'id', None)}, path: {path}" 135 ) 136 path.parent.mkdir(parents=True, exist_ok=True) 137 json_data = self.model_dump_json(indent=2, exclude={"path"}) 138 with open(path, "w") as file: 139 file.write(json_data) 140 # save the path so even if something like name changes, the file doesn't move 141 self.path = path 142 143 def build_path(self) -> Path | None: 144 if self.path is not None: 145 return self.path 146 return None 147 148 # increment for breaking changes 149 def max_schema_version(self) -> int: 150 return 1 151 152 153class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta): 154 """Base model for Kiln models that have a parent-child relationship. This base class is for child models. 155 156 This class provides functionality for managing hierarchical relationships between models, 157 including parent reference handling and file system organization. 158 159 Attributes: 160 _parent (KilnBaseModel): Reference to the parent model instance 161 """ 162 163 _parent: KilnBaseModel | None = None 164 165 # workaround to tell typechecker that we support the parent property, even though it's not a stock property 166 if TYPE_CHECKING: 167 parent: KilnBaseModel # type: ignore 168 169 def __init__(self, **data): 170 super().__init__(**data) 171 if "parent" in data: 172 self.parent = data["parent"] 173 174 @property 175 def parent(self) -> Optional[KilnBaseModel]: 176 """Get the parent model instance, loading it from disk if necessary. 177 178 Returns: 179 Optional[KilnBaseModel]: The parent model instance or None if not set 180 """ 181 if self._parent is not None: 182 return self._parent 183 # lazy load parent from path 184 if self.path is None: 185 return None 186 # TODO: this only works with base_filename. If we every support custom names, we need to change this. 187 parent_path = ( 188 self.path.parent.parent.parent 189 / self.__class__.parent_type().base_filename() 190 ) 191 if parent_path is None: 192 return None 193 self._parent = self.__class__.parent_type().load_from_file(parent_path) 194 return self._parent 195 196 @parent.setter 197 def parent(self, value: Optional[KilnBaseModel]): 198 if value is not None: 199 expected_parent_type = self.__class__.parent_type() 200 if not isinstance(value, expected_parent_type): 201 raise ValueError( 202 f"Parent must be of type {expected_parent_type}, but was {type(value)}" 203 ) 204 self._parent = value 205 206 # Dynamically implemented by KilnParentModel method injection 207 @classmethod 208 def relationship_name(cls) -> str: 209 raise NotImplementedError("Relationship name must be implemented") 210 211 # Dynamically implemented by KilnParentModel method injection 212 @classmethod 213 def parent_type(cls) -> Type[KilnBaseModel]: 214 raise NotImplementedError("Parent type must be implemented") 215 216 @model_validator(mode="after") 217 def check_parent_type(self) -> Self: 218 if self._parent is not None: 219 expected_parent_type = self.__class__.parent_type() 220 if not isinstance(self._parent, expected_parent_type): 221 raise ValueError( 222 f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}" 223 ) 224 return self 225 226 def build_child_dirname(self) -> Path: 227 # Default implementation for readable folder names. 228 # {id} - {name}/{type}.kiln 229 if self.id is None: 230 # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now. 231 raise ValueError("ID is not set - can not save or build path") 232 path = self.id 233 name = getattr(self, "name", None) 234 if name is not None: 235 path = f"{path} - {name[:32]}" 236 return Path(path) 237 238 def build_path(self) -> Path | None: 239 # if specifically loaded from an existing path, keep that no matter what 240 # this ensures the file structure is easy to use with git/version control 241 # and that changes to things like name (which impacts default path) don't leave dangling files 242 if self.path is not None: 243 return self.path 244 # Build a path under parent_folder/relationship/file.kiln 245 if self.parent is None: 246 return None 247 parent_path = self.parent.build_path() 248 if parent_path is None: 249 return None 250 parent_folder = parent_path.parent 251 if parent_folder is None: 252 return None 253 return ( 254 parent_folder 255 / self.__class__.relationship_name() 256 / self.build_child_dirname() 257 / self.__class__.base_filename() 258 ) 259 260 @classmethod 261 def all_children_of_parent_path( 262 cls: Type[PT], parent_path: Path | None 263 ) -> list[PT]: 264 if parent_path is None: 265 # children are disk based. If not saved, they don't exist 266 return [] 267 268 # Determine the parent folder 269 if parent_path.is_file(): 270 parent_folder = parent_path.parent 271 else: 272 parent_folder = parent_path 273 274 parent = cls.parent_type().load_from_file(parent_path) 275 if parent is None: 276 raise ValueError("Parent must be set to load children") 277 278 # Ignore type error: this is abstract base class, but children must implement relationship_name 279 relationship_folder = parent_folder / Path(cls.relationship_name()) # type: ignore 280 281 if not relationship_folder.exists() or not relationship_folder.is_dir(): 282 return [] 283 284 # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder 285 children = [] 286 for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"): 287 child = cls.load_from_file(child_file) 288 children.append(child) 289 290 return children 291 292 293# Parent create methods for all child relationships 294# You must pass in parent_of in the subclass definition, defining the child relationships 295class KilnParentModel(KilnBaseModel, metaclass=ABCMeta): 296 """Base model for Kiln models that can have child models. 297 298 This class provides functionality for managing collections of child models and their persistence. 299 Child relationships must be defined using the parent_of parameter in the class definition. 300 301 Args: 302 parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types 303 """ 304 305 @classmethod 306 def _create_child_method( 307 cls, relationship_name: str, child_class: Type[KilnParentedModel] 308 ): 309 def child_method(self) -> list[child_class]: 310 return child_class.all_children_of_parent_path(self.path) 311 312 child_method.__name__ = relationship_name 313 child_method.__annotations__ = {"return": List[child_class]} 314 setattr(cls, relationship_name, child_method) 315 316 @classmethod 317 def _create_parent_methods( 318 cls, targetCls: Type[KilnParentedModel], relationship_name: str 319 ): 320 def parent_class_method() -> Type[KilnParentModel]: 321 return cls 322 323 parent_class_method.__name__ = "parent_type" 324 parent_class_method.__annotations__ = {"return": Type[KilnParentModel]} 325 setattr(targetCls, "parent_type", parent_class_method) 326 327 def relationship_name_method() -> str: 328 return relationship_name 329 330 relationship_name_method.__name__ = "relationship_name" 331 relationship_name_method.__annotations__ = {"return": str} 332 setattr(targetCls, "relationship_name", relationship_name_method) 333 334 @classmethod 335 def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs): 336 super().__init_subclass__(**kwargs) 337 cls._parent_of = parent_of 338 for relationship_name, child_class in parent_of.items(): 339 cls._create_child_method(relationship_name, child_class) 340 cls._create_parent_methods(child_class, relationship_name) 341 342 @classmethod 343 def validate_and_save_with_subrelations( 344 cls, 345 data: Dict[str, Any], 346 path: Path | None = None, 347 parent: KilnBaseModel | None = None, 348 ): 349 """Validate and save a model instance along with all its nested child relationships. 350 351 Args: 352 data (Dict[str, Any]): Model data including child relationships 353 path (Path, optional): Path where the model should be saved 354 parent (KilnBaseModel, optional): Parent model instance for parented models 355 356 Returns: 357 KilnParentModel: The validated and saved model instance 358 359 Raises: 360 ValidationError: If validation fails for the model or any of its children 361 """ 362 # Validate first, then save. Don't want error half way through, and partly persisted 363 # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later. 364 cls._validate_nested(data, save=False, path=path, parent=parent) 365 instance = cls._validate_nested(data, save=True, path=path, parent=parent) 366 return instance 367 368 @classmethod 369 def _validate_nested( 370 cls, 371 data: Dict[str, Any], 372 save: bool = False, 373 parent: KilnBaseModel | None = None, 374 path: Path | None = None, 375 ): 376 # Collect all validation errors so we can report them all at once 377 validation_errors = [] 378 379 try: 380 instance = cls.model_validate(data, strict=True) 381 if path is not None: 382 instance.path = path 383 if parent is not None and isinstance(instance, KilnParentedModel): 384 instance.parent = parent 385 if save: 386 instance.save_to_file() 387 except ValidationError as e: 388 instance = None 389 for suberror in e.errors(): 390 validation_errors.append(suberror) 391 392 for key, value_list in data.items(): 393 if key in cls._parent_of: 394 parent_type = cls._parent_of[key] 395 if not isinstance(value_list, list): 396 raise ValueError( 397 f"Expected a list for {key}, but got {type(value_list)}" 398 ) 399 for value_index, value in enumerate(value_list): 400 try: 401 if issubclass(parent_type, KilnParentModel): 402 kwargs = {"data": value, "save": save} 403 if instance is not None: 404 kwargs["parent"] = instance 405 parent_type._validate_nested(**kwargs) 406 elif issubclass(parent_type, KilnParentedModel): 407 # Root node 408 subinstance = parent_type.model_validate(value, strict=True) 409 if instance is not None: 410 subinstance.parent = instance 411 if save: 412 subinstance.save_to_file() 413 else: 414 raise ValueError( 415 f"Invalid type {parent_type}. Should be KilnBaseModel based." 416 ) 417 except ValidationError as e: 418 for suberror in e.errors(): 419 cls._append_loc(suberror, key, value_index) 420 validation_errors.append(suberror) 421 422 if len(validation_errors) > 0: 423 raise ValidationError.from_exception_data( 424 title=f"Validation failed for {cls.__name__}", 425 line_errors=validation_errors, 426 input_type="json", 427 ) 428 429 return instance 430 431 @classmethod 432 def _append_loc( 433 cls, error: ErrorDetails, current_loc: str, value_index: int | None = None 434 ): 435 orig_loc = error["loc"] if "loc" in error else None 436 new_loc: list[str | int] = [current_loc] 437 if value_index is not None: 438 new_loc.append(value_index) 439 if isinstance(orig_loc, tuple): 440 new_loc.extend(list(orig_loc)) 441 elif isinstance(orig_loc, list): 442 new_loc.extend(orig_loc) 443 error["loc"] = tuple(new_loc)
42class KilnBaseModel(BaseModel): 43 """Base model for all Kiln data models with common functionality for persistence and versioning. 44 45 Attributes: 46 v (int): Schema version number for migration support 47 id (str): Unique identifier for the model instance 48 path (Path): File system path where the model is stored 49 created_at (datetime): Timestamp when the model was created 50 created_by (str): User ID of the creator 51 """ 52 53 model_config = ConfigDict(validate_assignment=True) 54 55 v: int = Field(default=1) # schema_version 56 id: ID_TYPE = ID_FIELD 57 path: Optional[Path] = Field(default=None) 58 created_at: datetime = Field(default_factory=datetime.now) 59 created_by: str = Field(default_factory=lambda: Config.shared().user_id) 60 61 @computed_field() 62 def model_type(self) -> str: 63 return self.type_name() 64 65 # if changing the model name, should keep the original name here for parsing old files 66 @classmethod 67 def type_name(cls) -> str: 68 return snake_case(cls.__name__) 69 70 # used as /obj_folder/base_filename.kiln 71 @classmethod 72 def base_filename(cls) -> str: 73 return cls.type_name() + ".kiln" 74 75 @classmethod 76 def load_from_folder(cls: Type[T], folderPath: Path) -> T: 77 """Load a model instance from a folder using the default filename. 78 79 Args: 80 folderPath (Path): Directory path containing the model file 81 82 Returns: 83 T: Instance of the model 84 """ 85 path = folderPath / cls.base_filename() 86 return cls.load_from_file(path) 87 88 @classmethod 89 def load_from_file(cls: Type[T], path: Path) -> T: 90 """Load a model instance from a specific file path. 91 92 Args: 93 path (Path): Path to the model file 94 95 Returns: 96 T: Instance of the model 97 98 Raises: 99 ValueError: If the loaded model is not of the expected type or version 100 """ 101 with open(path, "r") as file: 102 file_data = file.read() 103 # TODO P2 perf: parsing the JSON twice here. 104 # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums. 105 parsed_json = json.loads(file_data) 106 m = cls.model_validate_json(file_data, strict=True) 107 if not isinstance(m, cls): 108 raise ValueError(f"Loaded model is not of type {cls.__name__}") 109 file_data = None 110 m.path = path 111 if m.v > m.max_schema_version(): 112 raise ValueError( 113 f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. " 114 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 115 f"version: {m.v}, max version: {m.max_schema_version()}" 116 ) 117 if parsed_json["model_type"] != cls.type_name(): 118 raise ValueError( 119 f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. " 120 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 121 f"version: {m.v}, max version: {m.max_schema_version()}" 122 ) 123 return m 124 125 def save_to_file(self) -> None: 126 """Save the model instance to a file. 127 128 Raises: 129 ValueError: If the path is not set 130 """ 131 path = self.build_path() 132 if path is None: 133 raise ValueError( 134 f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, " 135 f"id: {getattr(self, 'id', None)}, path: {path}" 136 ) 137 path.parent.mkdir(parents=True, exist_ok=True) 138 json_data = self.model_dump_json(indent=2, exclude={"path"}) 139 with open(path, "w") as file: 140 file.write(json_data) 141 # save the path so even if something like name changes, the file doesn't move 142 self.path = path 143 144 def build_path(self) -> Path | None: 145 if self.path is not None: 146 return self.path 147 return None 148 149 # increment for breaking changes 150 def max_schema_version(self) -> int: 151 return 1
Base model for all Kiln data models with common functionality for persistence and versioning.
Attributes: v (int): Schema version number for migration support id (str): Unique identifier for the model instance path (Path): File system path where the model is stored created_at (datetime): Timestamp when the model was created created_by (str): User ID of the creator
75 @classmethod 76 def load_from_folder(cls: Type[T], folderPath: Path) -> T: 77 """Load a model instance from a folder using the default filename. 78 79 Args: 80 folderPath (Path): Directory path containing the model file 81 82 Returns: 83 T: Instance of the model 84 """ 85 path = folderPath / cls.base_filename() 86 return cls.load_from_file(path)
Load a model instance from a folder using the default filename.
Args: folderPath (Path): Directory path containing the model file
Returns: T: Instance of the model
88 @classmethod 89 def load_from_file(cls: Type[T], path: Path) -> T: 90 """Load a model instance from a specific file path. 91 92 Args: 93 path (Path): Path to the model file 94 95 Returns: 96 T: Instance of the model 97 98 Raises: 99 ValueError: If the loaded model is not of the expected type or version 100 """ 101 with open(path, "r") as file: 102 file_data = file.read() 103 # TODO P2 perf: parsing the JSON twice here. 104 # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums. 105 parsed_json = json.loads(file_data) 106 m = cls.model_validate_json(file_data, strict=True) 107 if not isinstance(m, cls): 108 raise ValueError(f"Loaded model is not of type {cls.__name__}") 109 file_data = None 110 m.path = path 111 if m.v > m.max_schema_version(): 112 raise ValueError( 113 f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. " 114 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 115 f"version: {m.v}, max version: {m.max_schema_version()}" 116 ) 117 if parsed_json["model_type"] != cls.type_name(): 118 raise ValueError( 119 f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. " 120 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 121 f"version: {m.v}, max version: {m.max_schema_version()}" 122 ) 123 return m
Load a model instance from a specific file path.
Args: path (Path): Path to the model file
Returns: T: Instance of the model
Raises: ValueError: If the loaded model is not of the expected type or version
125 def save_to_file(self) -> None: 126 """Save the model instance to a file. 127 128 Raises: 129 ValueError: If the path is not set 130 """ 131 path = self.build_path() 132 if path is None: 133 raise ValueError( 134 f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, " 135 f"id: {getattr(self, 'id', None)}, path: {path}" 136 ) 137 path.parent.mkdir(parents=True, exist_ok=True) 138 json_data = self.model_dump_json(indent=2, exclude={"path"}) 139 with open(path, "w") as file: 140 file.write(json_data) 141 # save the path so even if something like name changes, the file doesn't move 142 self.path = path
Save the model instance to a file.
Raises: ValueError: If the path is not set
154class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta): 155 """Base model for Kiln models that have a parent-child relationship. This base class is for child models. 156 157 This class provides functionality for managing hierarchical relationships between models, 158 including parent reference handling and file system organization. 159 160 Attributes: 161 _parent (KilnBaseModel): Reference to the parent model instance 162 """ 163 164 _parent: KilnBaseModel | None = None 165 166 # workaround to tell typechecker that we support the parent property, even though it's not a stock property 167 if TYPE_CHECKING: 168 parent: KilnBaseModel # type: ignore 169 170 def __init__(self, **data): 171 super().__init__(**data) 172 if "parent" in data: 173 self.parent = data["parent"] 174 175 @property 176 def parent(self) -> Optional[KilnBaseModel]: 177 """Get the parent model instance, loading it from disk if necessary. 178 179 Returns: 180 Optional[KilnBaseModel]: The parent model instance or None if not set 181 """ 182 if self._parent is not None: 183 return self._parent 184 # lazy load parent from path 185 if self.path is None: 186 return None 187 # TODO: this only works with base_filename. If we every support custom names, we need to change this. 188 parent_path = ( 189 self.path.parent.parent.parent 190 / self.__class__.parent_type().base_filename() 191 ) 192 if parent_path is None: 193 return None 194 self._parent = self.__class__.parent_type().load_from_file(parent_path) 195 return self._parent 196 197 @parent.setter 198 def parent(self, value: Optional[KilnBaseModel]): 199 if value is not None: 200 expected_parent_type = self.__class__.parent_type() 201 if not isinstance(value, expected_parent_type): 202 raise ValueError( 203 f"Parent must be of type {expected_parent_type}, but was {type(value)}" 204 ) 205 self._parent = value 206 207 # Dynamically implemented by KilnParentModel method injection 208 @classmethod 209 def relationship_name(cls) -> str: 210 raise NotImplementedError("Relationship name must be implemented") 211 212 # Dynamically implemented by KilnParentModel method injection 213 @classmethod 214 def parent_type(cls) -> Type[KilnBaseModel]: 215 raise NotImplementedError("Parent type must be implemented") 216 217 @model_validator(mode="after") 218 def check_parent_type(self) -> Self: 219 if self._parent is not None: 220 expected_parent_type = self.__class__.parent_type() 221 if not isinstance(self._parent, expected_parent_type): 222 raise ValueError( 223 f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}" 224 ) 225 return self 226 227 def build_child_dirname(self) -> Path: 228 # Default implementation for readable folder names. 229 # {id} - {name}/{type}.kiln 230 if self.id is None: 231 # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now. 232 raise ValueError("ID is not set - can not save or build path") 233 path = self.id 234 name = getattr(self, "name", None) 235 if name is not None: 236 path = f"{path} - {name[:32]}" 237 return Path(path) 238 239 def build_path(self) -> Path | None: 240 # if specifically loaded from an existing path, keep that no matter what 241 # this ensures the file structure is easy to use with git/version control 242 # and that changes to things like name (which impacts default path) don't leave dangling files 243 if self.path is not None: 244 return self.path 245 # Build a path under parent_folder/relationship/file.kiln 246 if self.parent is None: 247 return None 248 parent_path = self.parent.build_path() 249 if parent_path is None: 250 return None 251 parent_folder = parent_path.parent 252 if parent_folder is None: 253 return None 254 return ( 255 parent_folder 256 / self.__class__.relationship_name() 257 / self.build_child_dirname() 258 / self.__class__.base_filename() 259 ) 260 261 @classmethod 262 def all_children_of_parent_path( 263 cls: Type[PT], parent_path: Path | None 264 ) -> list[PT]: 265 if parent_path is None: 266 # children are disk based. If not saved, they don't exist 267 return [] 268 269 # Determine the parent folder 270 if parent_path.is_file(): 271 parent_folder = parent_path.parent 272 else: 273 parent_folder = parent_path 274 275 parent = cls.parent_type().load_from_file(parent_path) 276 if parent is None: 277 raise ValueError("Parent must be set to load children") 278 279 # Ignore type error: this is abstract base class, but children must implement relationship_name 280 relationship_folder = parent_folder / Path(cls.relationship_name()) # type: ignore 281 282 if not relationship_folder.exists() or not relationship_folder.is_dir(): 283 return [] 284 285 # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder 286 children = [] 287 for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"): 288 child = cls.load_from_file(child_file) 289 children.append(child) 290 291 return children
Base model for Kiln models that have a parent-child relationship. This base class is for child models.
This class provides functionality for managing hierarchical relationships between models, including parent reference handling and file system organization.
Attributes: _parent (KilnBaseModel): Reference to the parent model instance
170 def __init__(self, **data): 171 super().__init__(**data) 172 if "parent" in data: 173 self.parent = data["parent"]
Create a new model by parsing and validating input data from keyword arguments.
Raises [ValidationError
][pydantic_core.ValidationError] if the input data cannot be
validated to form a valid model.
self
is explicitly positional-only to allow self
as a field name.
175 @property 176 def parent(self) -> Optional[KilnBaseModel]: 177 """Get the parent model instance, loading it from disk if necessary. 178 179 Returns: 180 Optional[KilnBaseModel]: The parent model instance or None if not set 181 """ 182 if self._parent is not None: 183 return self._parent 184 # lazy load parent from path 185 if self.path is None: 186 return None 187 # TODO: this only works with base_filename. If we every support custom names, we need to change this. 188 parent_path = ( 189 self.path.parent.parent.parent 190 / self.__class__.parent_type().base_filename() 191 ) 192 if parent_path is None: 193 return None 194 self._parent = self.__class__.parent_type().load_from_file(parent_path) 195 return self._parent
Get the parent model instance, loading it from disk if necessary.
Returns: Optional[KilnBaseModel]: The parent model instance or None if not set
217 @model_validator(mode="after") 218 def check_parent_type(self) -> Self: 219 if self._parent is not None: 220 expected_parent_type = self.__class__.parent_type() 221 if not isinstance(self._parent, expected_parent_type): 222 raise ValueError( 223 f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}" 224 ) 225 return self
227 def build_child_dirname(self) -> Path: 228 # Default implementation for readable folder names. 229 # {id} - {name}/{type}.kiln 230 if self.id is None: 231 # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now. 232 raise ValueError("ID is not set - can not save or build path") 233 path = self.id 234 name = getattr(self, "name", None) 235 if name is not None: 236 path = f"{path} - {name[:32]}" 237 return Path(path)
239 def build_path(self) -> Path | None: 240 # if specifically loaded from an existing path, keep that no matter what 241 # this ensures the file structure is easy to use with git/version control 242 # and that changes to things like name (which impacts default path) don't leave dangling files 243 if self.path is not None: 244 return self.path 245 # Build a path under parent_folder/relationship/file.kiln 246 if self.parent is None: 247 return None 248 parent_path = self.parent.build_path() 249 if parent_path is None: 250 return None 251 parent_folder = parent_path.parent 252 if parent_folder is None: 253 return None 254 return ( 255 parent_folder 256 / self.__class__.relationship_name() 257 / self.build_child_dirname() 258 / self.__class__.base_filename() 259 )
261 @classmethod 262 def all_children_of_parent_path( 263 cls: Type[PT], parent_path: Path | None 264 ) -> list[PT]: 265 if parent_path is None: 266 # children are disk based. If not saved, they don't exist 267 return [] 268 269 # Determine the parent folder 270 if parent_path.is_file(): 271 parent_folder = parent_path.parent 272 else: 273 parent_folder = parent_path 274 275 parent = cls.parent_type().load_from_file(parent_path) 276 if parent is None: 277 raise ValueError("Parent must be set to load children") 278 279 # Ignore type error: this is abstract base class, but children must implement relationship_name 280 relationship_folder = parent_folder / Path(cls.relationship_name()) # type: ignore 281 282 if not relationship_folder.exists() or not relationship_folder.is_dir(): 283 return [] 284 285 # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder 286 children = [] 287 for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"): 288 child = cls.load_from_file(child_file) 289 children.append(child) 290 291 return children
281def init_private_attributes(self: BaseModel, context: Any, /) -> None: 282 """This function is meant to behave like a BaseModel method to initialise private attributes. 283 284 It takes context as an argument since that's what pydantic-core passes when calling it. 285 286 Args: 287 self: The BaseModel instance. 288 context: The context. 289 """ 290 if getattr(self, '__pydantic_private__', None) is None: 291 pydantic_private = {} 292 for name, private_attr in self.__private_attributes__.items(): 293 default = private_attr.get_default() 294 if default is not PydanticUndefined: 295 pydantic_private[name] = default 296 object_setattr(self, '__pydantic_private__', pydantic_private)
This function is meant to behave like a BaseModel method to initialise private attributes.
It takes context as an argument since that's what pydantic-core passes when calling it.
Args: self: The BaseModel instance. context: The context.
296class KilnParentModel(KilnBaseModel, metaclass=ABCMeta): 297 """Base model for Kiln models that can have child models. 298 299 This class provides functionality for managing collections of child models and their persistence. 300 Child relationships must be defined using the parent_of parameter in the class definition. 301 302 Args: 303 parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types 304 """ 305 306 @classmethod 307 def _create_child_method( 308 cls, relationship_name: str, child_class: Type[KilnParentedModel] 309 ): 310 def child_method(self) -> list[child_class]: 311 return child_class.all_children_of_parent_path(self.path) 312 313 child_method.__name__ = relationship_name 314 child_method.__annotations__ = {"return": List[child_class]} 315 setattr(cls, relationship_name, child_method) 316 317 @classmethod 318 def _create_parent_methods( 319 cls, targetCls: Type[KilnParentedModel], relationship_name: str 320 ): 321 def parent_class_method() -> Type[KilnParentModel]: 322 return cls 323 324 parent_class_method.__name__ = "parent_type" 325 parent_class_method.__annotations__ = {"return": Type[KilnParentModel]} 326 setattr(targetCls, "parent_type", parent_class_method) 327 328 def relationship_name_method() -> str: 329 return relationship_name 330 331 relationship_name_method.__name__ = "relationship_name" 332 relationship_name_method.__annotations__ = {"return": str} 333 setattr(targetCls, "relationship_name", relationship_name_method) 334 335 @classmethod 336 def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs): 337 super().__init_subclass__(**kwargs) 338 cls._parent_of = parent_of 339 for relationship_name, child_class in parent_of.items(): 340 cls._create_child_method(relationship_name, child_class) 341 cls._create_parent_methods(child_class, relationship_name) 342 343 @classmethod 344 def validate_and_save_with_subrelations( 345 cls, 346 data: Dict[str, Any], 347 path: Path | None = None, 348 parent: KilnBaseModel | None = None, 349 ): 350 """Validate and save a model instance along with all its nested child relationships. 351 352 Args: 353 data (Dict[str, Any]): Model data including child relationships 354 path (Path, optional): Path where the model should be saved 355 parent (KilnBaseModel, optional): Parent model instance for parented models 356 357 Returns: 358 KilnParentModel: The validated and saved model instance 359 360 Raises: 361 ValidationError: If validation fails for the model or any of its children 362 """ 363 # Validate first, then save. Don't want error half way through, and partly persisted 364 # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later. 365 cls._validate_nested(data, save=False, path=path, parent=parent) 366 instance = cls._validate_nested(data, save=True, path=path, parent=parent) 367 return instance 368 369 @classmethod 370 def _validate_nested( 371 cls, 372 data: Dict[str, Any], 373 save: bool = False, 374 parent: KilnBaseModel | None = None, 375 path: Path | None = None, 376 ): 377 # Collect all validation errors so we can report them all at once 378 validation_errors = [] 379 380 try: 381 instance = cls.model_validate(data, strict=True) 382 if path is not None: 383 instance.path = path 384 if parent is not None and isinstance(instance, KilnParentedModel): 385 instance.parent = parent 386 if save: 387 instance.save_to_file() 388 except ValidationError as e: 389 instance = None 390 for suberror in e.errors(): 391 validation_errors.append(suberror) 392 393 for key, value_list in data.items(): 394 if key in cls._parent_of: 395 parent_type = cls._parent_of[key] 396 if not isinstance(value_list, list): 397 raise ValueError( 398 f"Expected a list for {key}, but got {type(value_list)}" 399 ) 400 for value_index, value in enumerate(value_list): 401 try: 402 if issubclass(parent_type, KilnParentModel): 403 kwargs = {"data": value, "save": save} 404 if instance is not None: 405 kwargs["parent"] = instance 406 parent_type._validate_nested(**kwargs) 407 elif issubclass(parent_type, KilnParentedModel): 408 # Root node 409 subinstance = parent_type.model_validate(value, strict=True) 410 if instance is not None: 411 subinstance.parent = instance 412 if save: 413 subinstance.save_to_file() 414 else: 415 raise ValueError( 416 f"Invalid type {parent_type}. Should be KilnBaseModel based." 417 ) 418 except ValidationError as e: 419 for suberror in e.errors(): 420 cls._append_loc(suberror, key, value_index) 421 validation_errors.append(suberror) 422 423 if len(validation_errors) > 0: 424 raise ValidationError.from_exception_data( 425 title=f"Validation failed for {cls.__name__}", 426 line_errors=validation_errors, 427 input_type="json", 428 ) 429 430 return instance 431 432 @classmethod 433 def _append_loc( 434 cls, error: ErrorDetails, current_loc: str, value_index: int | None = None 435 ): 436 orig_loc = error["loc"] if "loc" in error else None 437 new_loc: list[str | int] = [current_loc] 438 if value_index is not None: 439 new_loc.append(value_index) 440 if isinstance(orig_loc, tuple): 441 new_loc.extend(list(orig_loc)) 442 elif isinstance(orig_loc, list): 443 new_loc.extend(orig_loc) 444 error["loc"] = tuple(new_loc)
Base model for Kiln models that can have child models.
This class provides functionality for managing collections of child models and their persistence. Child relationships must be defined using the parent_of parameter in the class definition.
Args: parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types
343 @classmethod 344 def validate_and_save_with_subrelations( 345 cls, 346 data: Dict[str, Any], 347 path: Path | None = None, 348 parent: KilnBaseModel | None = None, 349 ): 350 """Validate and save a model instance along with all its nested child relationships. 351 352 Args: 353 data (Dict[str, Any]): Model data including child relationships 354 path (Path, optional): Path where the model should be saved 355 parent (KilnBaseModel, optional): Parent model instance for parented models 356 357 Returns: 358 KilnParentModel: The validated and saved model instance 359 360 Raises: 361 ValidationError: If validation fails for the model or any of its children 362 """ 363 # Validate first, then save. Don't want error half way through, and partly persisted 364 # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later. 365 cls._validate_nested(data, save=False, path=path, parent=parent) 366 instance = cls._validate_nested(data, save=True, path=path, parent=parent) 367 return instance
Validate and save a model instance along with all its nested child relationships.
Args: data (Dict[str, Any]): Model data including child relationships path (Path, optional): Path where the model should be saved parent (KilnBaseModel, optional): Parent model instance for parented models
Returns: KilnParentModel: The validated and saved model instance
Raises: ValidationError: If validation fails for the model or any of its children