Coverage for src/typedal/core.py: 100%
905 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-05-22 20:39 +0200
« prev ^ index » next coverage.py v7.4.1, created at 2024-05-22 20:39 +0200
1"""
2Core functionality of TypeDAL.
3"""
5import contextlib
6import csv
7import datetime as dt
8import inspect
9import json
10import math
11import types
12import typing
13import warnings
14from collections import defaultdict
15from copy import copy
16from decimal import Decimal
17from pathlib import Path
18from typing import Any, Optional
20import pydal
21from pydal._globals import DEFAULT
22from pydal.objects import Field as _Field
23from pydal.objects import Query as _Query
24from pydal.objects import Row
25from pydal.objects import Table as _Table
26from typing_extensions import Self, Unpack
28from .config import TypeDALConfig, load_config
29from .helpers import (
30 DummyQuery,
31 all_annotations,
32 all_dict,
33 as_lambda,
34 extract_type_optional,
35 filter_out,
36 instanciate,
37 is_union,
38 looks_like,
39 mktable,
40 origin_is_subclass,
41 to_snake,
42 unwrap_type,
43)
44from .serializers import as_json
45from .types import (
46 AfterDeleteCallable,
47 AfterInsertCallable,
48 AfterUpdateCallable,
49 AnyDict,
50 BeforeDeleteCallable,
51 BeforeInsertCallable,
52 BeforeUpdateCallable,
53 CacheMetadata,
54 Expression,
55 Field,
56 Metadata,
57 PaginateDict,
58 Pagination,
59 Query,
60 Rows,
61 SelectKwargs,
62 Table,
63 Validator,
64 _Types,
65)
67# use typing.cast(type, ...) to make mypy happy with unions
68T_annotation = typing.Type[Any] | types.UnionType
69T_Query = typing.Union["Table", Query, bool, None, "TypedTable", typing.Type["TypedTable"]]
70T_Value = typing.TypeVar("T_Value") # actual type of the Field (via Generic)
71T_MetaInstance = typing.TypeVar("T_MetaInstance", bound="TypedTable") # bound="TypedTable"; bound="TableMeta"
72T = typing.TypeVar("T")
74BASIC_MAPPINGS: dict[T_annotation, str] = {
75 str: "string",
76 int: "integer",
77 bool: "boolean",
78 bytes: "blob",
79 float: "double",
80 object: "json",
81 Decimal: "decimal(10,2)",
82 dt.date: "date",
83 dt.time: "time",
84 dt.datetime: "datetime",
85}
88def is_typed_field(cls: Any) -> typing.TypeGuard["TypedField[Any]"]:
89 """
90 Is `cls` an instance or subclass of TypedField?
92 Deprecated
93 """
94 return (
95 isinstance(cls, TypedField)
96 or isinstance(typing.get_origin(cls), type)
97 and issubclass(typing.get_origin(cls), TypedField)
98 )
101JOIN_OPTIONS = typing.Literal["left", "inner", None]
102DEFAULT_JOIN_OPTION: JOIN_OPTIONS = "left"
104# table-ish paramter:
105P_Table = typing.Union[typing.Type["TypedTable"], pydal.objects.Table]
107Condition: typing.TypeAlias = typing.Optional[
108 typing.Callable[
109 # self, other -> Query
110 [P_Table, P_Table],
111 Query | bool,
112 ]
113]
115OnQuery: typing.TypeAlias = typing.Optional[
116 typing.Callable[
117 # self, other -> list of .on statements
118 [P_Table, P_Table],
119 list[Expression],
120 ]
121]
123To_Type = typing.TypeVar("To_Type", type[Any], typing.Type[Any], str)
126class Relationship(typing.Generic[To_Type]):
127 """
128 Define a relationship to another table.
129 """
131 _type: To_Type
132 table: typing.Type["TypedTable"] | type | str
133 condition: Condition
134 on: OnQuery
135 multiple: bool
136 join: JOIN_OPTIONS
138 def __init__(
139 self,
140 _type: To_Type,
141 condition: Condition = None,
142 join: JOIN_OPTIONS = None,
143 on: OnQuery = None,
144 ):
145 """
146 Should not be called directly, use relationship() instead!
147 """
148 if condition and on:
149 warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}")
150 raise ValueError("Please specify either a condition or an 'on' statement for this relationship!")
152 self._type = _type
153 self.condition = condition
154 self.join = "left" if on else join # .on is always left join!
155 self.on = on
157 if args := typing.get_args(_type):
158 self.table = unwrap_type(args[0])
159 self.multiple = True
160 else:
161 self.table = _type
162 self.multiple = False
164 if isinstance(self.table, str):
165 self.table = TypeDAL.to_snake(self.table)
167 def clone(self, **update: Any) -> "Relationship[To_Type]":
168 """
169 Create a copy of the relationship, possibly updated.
170 """
171 return self.__class__(
172 update.get("_type") or self._type,
173 update.get("condition") or self.condition,
174 update.get("join") or self.join,
175 update.get("on") or self.on,
176 )
178 def __repr__(self) -> str:
179 """
180 Representation of the relationship.
181 """
182 if callback := self.condition or self.on:
183 src_code = inspect.getsource(callback).strip()
184 else:
185 cls_name = self._type if isinstance(self._type, str) else self._type.__name__ # type: ignore
186 src_code = f"to {cls_name} (missing condition)"
188 join = f":{self.join}" if self.join else ""
189 return f"<Relationship{join} {src_code}>"
191 def get_table(self, db: "TypeDAL") -> typing.Type["TypedTable"]:
192 """
193 Get the table this relationship is bound to.
194 """
195 table = self.table # can be a string because db wasn't available yet
196 if isinstance(table, str):
197 if mapped := db._class_map.get(table):
198 # yay
199 return mapped
201 # boo, fall back to untyped table but pretend it is typed:
202 return typing.cast(typing.Type["TypedTable"], db[table]) # eh close enough!
204 return table
206 def get_table_name(self) -> str:
207 """
208 Get the name of the table this relationship is bound to.
209 """
210 if isinstance(self.table, str):
211 return self.table
213 if isinstance(self.table, pydal.objects.Table):
214 return str(self.table)
216 # else: typed table
217 try:
218 table = self.table._ensure_table_defined() if issubclass(self.table, TypedTable) else self.table
219 except Exception: # pragma: no cover
220 table = self.table
222 return str(table)
224 def __get__(self, instance: Any, owner: Any) -> typing.Optional[list[Any]] | "Relationship[To_Type]":
225 """
226 Relationship is a descriptor class, which can be returned from a class but not an instance.
228 For an instance, using .join() will replace the Relationship with the actual data.
229 If you forgot to join, a warning will be shown and empty data will be returned.
230 """
231 if not instance:
232 # relationship queried on class, that's allowed
233 return self
235 warnings.warn(
236 "Trying to get data from a relationship object! Did you forget to join it?", category=RuntimeWarning
237 )
238 if self.multiple:
239 return []
240 else:
241 return None
244def relationship(
245 _type: To_Type, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None
246) -> Relationship[To_Type]:
247 """
248 Define a relationship to another table, when its id is not stored in the current table.
250 Example:
251 class User(TypedTable):
252 name: str
254 posts = relationship(list["Post"], condition=lambda self, post: self.id == post.author, join='left')
256 class Post(TypedTable):
257 title: str
258 author: User
260 User.join("posts").first() # User instance with list[Post] in .posts
262 Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts.
263 In this case, the join strategy is set to LEFT so users without posts are also still selected.
265 For complex queries with a pivot table, a `on` can be set insteaad of `condition`:
266 class User(TypedTable):
267 ...
269 tags = relationship(list["Tag"], on=lambda self, tag: [
270 Tagged.on(Tagged.entity == entity.gid),
271 Tag.on((Tagged.tag == tag.id)),
272 ])
274 If you'd try to capture this in a single 'condition', pydal would create a cross join which is much less efficient.
275 """
276 return Relationship(_type, condition, join, on)
279def _generate_relationship_condition(
280 _: typing.Type["TypedTable"], key: str, field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]]
281) -> Condition:
282 origin = typing.get_origin(field)
283 # else: generic
285 if origin == list:
286 # field = typing.get_args(field)[0] # actual field
287 # return lambda _self, _other: cls[key].contains(field)
289 return lambda _self, _other: _self[key].contains(_other.id)
290 else:
291 # normal reference
292 # return lambda _self, _other: cls[key] == field.id
293 return lambda _self, _other: _self[key] == _other.id
296def to_relationship(
297 cls: typing.Type["TypedTable"] | type[Any],
298 key: str,
299 field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]],
300) -> typing.Optional[Relationship[Any]]:
301 """
302 Used to automatically create relationship instance for reference fields.
304 Example:
305 class MyTable(TypedTable):
306 reference: OtherTable
308 `reference` contains the id of an Other Table row.
309 MyTable.relationships should have 'reference' as a relationship, so `MyTable.join('reference')` should work.
311 This function will automatically perform this logic (called in db.define):
312 to_relationship(MyTable, 'reference', OtherTable) -> Relationship[OtherTable]
314 Also works for list:reference (list[OtherTable]) and TypedField[OtherTable].
315 """
316 if looks_like(field, TypedField):
317 if args := typing.get_args(field):
318 field = args[0]
319 else:
320 # weird
321 return None
323 field, optional = extract_type_optional(field)
325 try:
326 condition = _generate_relationship_condition(cls, key, field)
327 except Exception as e: # pragma: no cover
328 warnings.warn("Could not generate Relationship condition", source=e)
329 condition = None
331 if not condition: # pragma: no cover
332 # something went wrong, not a valid relationship
333 warnings.warn(f"Invalid relationship for {cls.__name__}.{key}: {field}")
334 return None
336 join = "left" if optional or typing.get_origin(field) == list else "inner"
338 return Relationship(typing.cast(type[TypedTable], field), condition, typing.cast(JOIN_OPTIONS, join))
341class TypeDAL(pydal.DAL): # type: ignore
342 """
343 Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables.
344 """
346 _config: TypeDALConfig
348 def __init__(
349 self,
350 uri: Optional[str] = None, # default from config or 'sqlite:memory'
351 pool_size: int = None, # default 1 if sqlite else 3
352 folder: Optional[str | Path] = None, # default 'databases' in config
353 db_codec: str = "UTF-8",
354 check_reserved: Optional[list[str]] = None,
355 migrate: Optional[bool] = None, # default True by config
356 fake_migrate: Optional[bool] = None, # default False by config
357 migrate_enabled: bool = True,
358 fake_migrate_all: bool = False,
359 decode_credentials: bool = False,
360 driver_args: Optional[AnyDict] = None,
361 adapter_args: Optional[AnyDict] = None,
362 attempts: int = 5,
363 auto_import: bool = False,
364 bigint_id: bool = False,
365 debug: bool = False,
366 lazy_tables: bool = False,
367 db_uid: Optional[str] = None,
368 after_connection: typing.Callable[..., Any] = None,
369 tables: Optional[list[str]] = None,
370 ignore_field_case: bool = True,
371 entity_quoting: bool = True,
372 table_hash: Optional[str] = None,
373 enable_typedal_caching: bool = None,
374 use_pyproject: bool | str = True,
375 use_env: bool | str = True,
376 connection: Optional[str] = None,
377 config: Optional[TypeDALConfig] = None,
378 ) -> None:
379 """
380 Adds some internal tables after calling pydal's default init.
382 Set enable_typedal_caching to False to disable this behavior.
383 """
384 config = config or load_config(connection, _use_pyproject=use_pyproject, _use_env=use_env)
385 config.update(
386 database=uri,
387 dialect=uri.split(":")[0] if uri and ":" in uri else None,
388 folder=str(folder) if folder is not None else None,
389 migrate=migrate,
390 fake_migrate=fake_migrate,
391 caching=enable_typedal_caching,
392 pool_size=pool_size,
393 )
395 self._config = config
397 if config.folder:
398 Path(config.folder).mkdir(exist_ok=True)
400 super().__init__(
401 config.database,
402 config.pool_size,
403 config.folder,
404 db_codec,
405 check_reserved,
406 config.migrate,
407 config.fake_migrate,
408 migrate_enabled,
409 fake_migrate_all,
410 decode_credentials,
411 driver_args,
412 adapter_args,
413 attempts,
414 auto_import,
415 bigint_id,
416 debug,
417 lazy_tables,
418 db_uid,
419 after_connection,
420 tables,
421 ignore_field_case,
422 entity_quoting,
423 table_hash,
424 )
426 if config.caching:
427 self.try_define(_TypedalCache)
428 self.try_define(_TypedalCacheDependency)
430 def try_define(self, model: typing.Type[T], verbose: bool = False) -> typing.Type[T]:
431 """
432 Try to define a model with migrate or fall back to fake migrate.
433 """
434 try:
435 return self.define(model, migrate=True)
436 except Exception as e:
437 # clean up:
438 self.rollback()
439 if (tablename := self.to_snake(model.__name__)) and tablename in dir(self):
440 delattr(self, tablename)
442 if verbose:
443 warnings.warn(f"{model} could not be migrated, try faking", source=e, category=RuntimeWarning)
445 # try again:
446 return self.define(model, migrate=True, fake_migrate=True, redefine=True)
448 default_kwargs: typing.ClassVar[AnyDict] = {
449 # fields are 'required' (notnull) by default:
450 "notnull": True,
451 }
453 # maps table name to typedal class, for resolving future references
454 _class_map: typing.ClassVar[dict[str, typing.Type["TypedTable"]]] = {}
456 def _define(self, cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:
457 # todo: new relationship item added should also invalidate (previously unrelated) cache result
459 # todo: option to enable/disable cache dependency behavior:
460 # - don't set _before_update and _before_delete
461 # - don't add TypedalCacheDependency entry
462 # - don't invalidate other item on new row of this type
464 # when __future__.annotations is implemented, cls.__annotations__ will not work anymore as below.
465 # proper way to handle this would be (but gives error right now due to Table implementing magic methods):
466 # typing.get_type_hints(cls, globalns=None, localns=None)
468 # dirty way (with evil eval):
469 # [eval(v) for k, v in cls.__annotations__.items()]
470 # this however also stops working when variables outside this scope or even references to other
471 # objects are used. So for now, this package will NOT work when from __future__ import annotations is used,
472 # and might break in the future, when this annotations behavior is enabled by default.
474 # non-annotated variables have to be passed to define_table as kwargs
475 full_dict = all_dict(cls) # includes properties from parents (e.g. useful for mixins)
477 tablename = self.to_snake(cls.__name__)
478 # grab annotations of cls and it's parents:
479 annotations = all_annotations(cls)
480 # extend with `prop = TypedField()` 'annotations':
481 annotations |= {k: typing.cast(type, v) for k, v in full_dict.items() if is_typed_field(v)}
482 # remove internal stuff:
483 annotations = {k: v for k, v in annotations.items() if not k.startswith("_")}
485 typedfields: dict[str, TypedField[Any]] = {
486 k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v)
487 }
489 relationships: dict[str, type[Relationship[Any]]] = filter_out(annotations, Relationship)
491 fields = {fname: self._to_field(fname, ftype) for fname, ftype in annotations.items()}
493 # ! dont' use full_dict here:
494 other_kwargs = kwargs | {
495 k: v for k, v in cls.__dict__.items() if k not in annotations and not k.startswith("_")
496 } # other_kwargs was previously used to pass kwargs to typedal, but use @define(**kwargs) for that.
497 # now it's only used to extract relationships from the object.
498 # other properties of the class (incl methods) should not be touched
500 # for key in typedfields.keys() - full_dict.keys():
501 # # typed fields that don't haven't been added to the object yet
502 # setattr(cls, key, typedfields[key])
504 for key, field in typedfields.items():
505 # clone every property so it can be re-used across mixins:
506 clone = copy(field)
507 setattr(cls, key, clone)
508 typedfields[key] = clone
510 # start with base classes and overwrite with current class:
511 relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship)
513 # DEPRECATED: Relationship as annotation is currently not supported!
514 # ensure they are all instances and
515 # not mix of instances (`= relationship()`) and classes (`: Relationship[...]`):
516 # relationships = {
517 # k: v if isinstance(v, Relationship) else to_relationship(cls, k, v) for k, v in relationships.items()
518 # }
520 # keys of implicit references (also relationships):
521 reference_field_keys = [k for k, v in fields.items() if v.type.split(" ")[0] in ("list:reference", "reference")]
523 # add implicit relationships:
524 # User; list[User]; TypedField[User]; TypedField[list[User]]
525 relationships |= {
526 k: new_relationship
527 for k in reference_field_keys
528 if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k]))
529 }
531 cache_dependency = self._config.caching and kwargs.pop("cache_dependency", True)
533 table: Table = self.define_table(tablename, *fields.values(), **kwargs)
535 for name, typed_field in typedfields.items():
536 field = fields[name]
537 typed_field.bind(field, table)
539 if issubclass(cls, TypedTable):
540 cls.__set_internals__(
541 db=self,
542 table=table,
543 # by now, all relationships should be instances!
544 relationships=typing.cast(dict[str, Relationship[Any]], relationships),
545 )
546 self._class_map[str(table)] = cls
547 cls.__on_define__(self)
548 else:
549 warnings.warn("db.define used without inheriting TypedTable. This could lead to strange problems!")
551 if not tablename.startswith("typedal_") and cache_dependency:
552 table._before_update.append(lambda s, _: _remove_cache(s, tablename))
553 table._before_delete.append(lambda s: _remove_cache(s, tablename))
555 return cls
557 @typing.overload
558 def define(self, maybe_cls: None = None, **kwargs: Any) -> typing.Callable[[typing.Type[T]], typing.Type[T]]:
559 """
560 Typing Overload for define without a class.
562 @db.define()
563 class MyTable(TypedTable): ...
564 """
566 @typing.overload
567 def define(self, maybe_cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:
568 """
569 Typing Overload for define with a class.
571 @db.define
572 class MyTable(TypedTable): ...
573 """
575 def define(
576 self, maybe_cls: typing.Type[T] | None = None, **kwargs: Any
577 ) -> typing.Type[T] | typing.Callable[[typing.Type[T]], typing.Type[T]]:
578 """
579 Can be used as a decorator on a class that inherits `TypedTable`, \
580 or as a regular method if you need to define your classes before you have access to a 'db' instance.
582 You can also pass extra arguments to db.define_table.
583 See http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Table-constructor
585 Example:
586 @db.define
587 class Person(TypedTable):
588 ...
590 class Article(TypedTable):
591 ...
593 # at a later time:
594 db.define(Article)
596 Returns:
597 the result of pydal.define_table
598 """
600 def wrapper(cls: typing.Type[T]) -> typing.Type[T]:
601 return self._define(cls, **kwargs)
603 if maybe_cls:
604 return wrapper(maybe_cls)
606 return wrapper
608 # def drop(self, table_name: str) -> None:
609 # """
610 # Remove a table by name (both on the database level and the typedal level).
611 # """
612 # # drop calls TypedTable.drop() and removes it from the `_class_map`
613 # if cls := self._class_map.pop(table_name, None):
614 # cls.drop()
616 # def drop_all(self, max_retries: int = None) -> None:
617 # """
618 # Remove all tables and keep doing so until everything is gone!
619 # """
620 # retries = 0
621 # if max_retries is None:
622 # max_retries = len(self.tables)
623 #
624 # while self.tables:
625 # retries += 1
626 # for table in self.tables:
627 # self.drop(table)
628 #
629 # if retries > max_retries:
630 # raise RuntimeError("Could not delete all tables")
632 def __call__(self, *_args: T_Query, **kwargs: Any) -> "TypedSet":
633 """
634 A db instance can be called directly to perform a query.
636 Usually, only a query is passed.
638 Example:
639 db(query).select()
641 """
642 args = list(_args)
643 if args:
644 cls = args[0]
645 if isinstance(cls, bool):
646 raise ValueError("Don't actually pass a bool to db()! Use a query instead.")
648 if isinstance(cls, type) and issubclass(type(cls), type) and issubclass(cls, TypedTable):
649 # table defined without @db.define decorator!
650 _cls: typing.Type[TypedTable] = cls
651 args[0] = _cls.id != None
653 _set = super().__call__(*args, **kwargs)
654 return typing.cast(TypedSet, _set)
656 def __getitem__(self, key: str) -> "Table":
657 """
658 Allows dynamically accessing a table by its name as a string.
660 Example:
661 db['users'] -> user
662 """
663 return typing.cast(Table, super().__getitem__(str(key)))
665 @classmethod
666 def _build_field(cls, name: str, _type: str, **kw: Any) -> Field:
667 return Field(name, _type, **{**cls.default_kwargs, **kw})
669 @classmethod
670 def _annotation_to_pydal_fieldtype(
671 cls, _ftype: T_annotation, mut_kw: typing.MutableMapping[str, Any]
672 ) -> Optional[str]:
673 # ftype can be a union or type. typing.cast is sometimes used to tell mypy when it's not a union.
674 ftype = typing.cast(type, _ftype) # cast from typing.Type to type to make mypy happy)
676 if isinstance(ftype, str):
677 # extract type from string
678 ftype = typing.get_args(typing.Type[ftype])[0]._evaluate(
679 localns=locals(), globalns=globals(), recursive_guard=frozenset()
680 )
682 if mapping := BASIC_MAPPINGS.get(ftype):
683 # basi types
684 return mapping
685 elif isinstance(ftype, _Table):
686 # db.table
687 return f"reference {ftype._tablename}"
688 elif issubclass(type(ftype), type) and issubclass(ftype, TypedTable):
689 # SomeTable
690 snakename = cls.to_snake(ftype.__name__)
691 return f"reference {snakename}"
692 elif isinstance(ftype, TypedField):
693 # FieldType(type, ...)
694 return ftype._to_field(mut_kw)
695 elif origin_is_subclass(ftype, TypedField):
696 # TypedField[int]
697 return cls._annotation_to_pydal_fieldtype(typing.get_args(ftype)[0], mut_kw)
698 elif isinstance(ftype, types.GenericAlias) and typing.get_origin(ftype) in (list, TypedField):
699 # list[str] -> str -> string -> list:string
700 _child_type = typing.get_args(ftype)[0]
701 _child_type = cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)
702 return f"list:{_child_type}"
703 elif is_union(ftype):
704 # str | int -> UnionType
705 # typing.Union[str | int] -> typing._UnionGenericAlias
707 # Optional[type] == type | None
709 match typing.get_args(ftype):
710 case (_child_type, _Types.NONETYPE) | (_Types.NONETYPE, _child_type):
711 # good union of Nullable
713 # if a field is optional, it is nullable:
714 mut_kw["notnull"] = False
715 return cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)
716 case _:
717 # two types is not supported by the db!
718 return None
719 else:
720 return None
722 @classmethod
723 def _to_field(cls, fname: str, ftype: type, **kw: Any) -> Field:
724 """
725 Convert a annotation into a pydal Field.
727 Args:
728 fname: name of the property
729 ftype: annotation of the property
730 kw: when using TypedField or a function returning it (e.g. StringField),
731 keyword args can be used to pass any other settings you would normally to a pydal Field
733 -> pydal.Field(fname, ftype, **kw)
735 Example:
736 class MyTable:
737 fname: ftype
738 id: int
739 name: str
740 reference: Table
741 other: TypedField(str, default="John Doe") # default will be in kwargs
742 """
743 fname = cls.to_snake(fname)
745 if converted_type := cls._annotation_to_pydal_fieldtype(ftype, kw):
746 return cls._build_field(fname, converted_type, **kw)
747 else:
748 raise NotImplementedError(f"Unsupported type {ftype}/{type(ftype)}")
750 @staticmethod
751 def to_snake(camel: str) -> str:
752 """
753 Moved to helpers, kept as a static method for legacy reasons.
754 """
755 return to_snake(camel)
758class TableMeta(type):
759 """
760 This metaclass contains functionality on table classes, that doesn't exist on its instances.
762 Example:
763 class MyTable(TypedTable):
764 some_field: TypedField[int]
766 MyTable.update_or_insert(...) # should work
768 MyTable.some_field # -> Field, can be used to query etc.
770 row = MyTable.first() # returns instance of MyTable
772 # row.update_or_insert(...) # shouldn't work!
774 row.some_field # -> int, with actual data
776 """
778 # set up by db.define:
779 # _db: TypeDAL | None = None
780 # _table: Table | None = None
781 _db: TypeDAL | None = None
782 _table: Table | None = None
783 _relationships: dict[str, Relationship[Any]] | None = None
785 #########################
786 # TypeDAL custom logic: #
787 #########################
789 def __set_internals__(self, db: pydal.DAL, table: Table, relationships: dict[str, Relationship[Any]]) -> None:
790 """
791 Store the related database and pydal table for later usage.
792 """
793 self._db = db
794 self._table = table
795 self._relationships = relationships
797 def __getattr__(self, col: str) -> Optional[Field]:
798 """
799 Magic method used by TypedTableMeta to get a database field with dot notation on a class.
801 Example:
802 SomeTypedTable.col -> db.table.col (via TypedTableMeta.__getattr__)
804 """
805 if self._table:
806 return getattr(self._table, col, None)
808 return None
810 def _ensure_table_defined(self) -> Table:
811 if not self._table:
812 raise EnvironmentError("@define or db.define is not called on this class yet!")
813 return self._table
815 def __iter__(self) -> typing.Generator[Field, None, None]:
816 """
817 Loop through the columns of this model.
818 """
819 table = self._ensure_table_defined()
820 yield from iter(table)
822 def __getitem__(self, item: str) -> Field:
823 """
824 Allow dict notation to get a column of this table (-> Field instance).
825 """
826 table = self._ensure_table_defined()
827 return table[item]
829 def __str__(self) -> str:
830 """
831 Normally, just returns the underlying table name, but with a fallback if the model is unbound.
832 """
833 if self._table:
834 return str(self._table)
835 else:
836 return f"<unbound table {self.__name__}>"
838 def from_row(self: typing.Type[T_MetaInstance], row: pydal.objects.Row) -> T_MetaInstance:
839 """
840 Create a model instance from a pydal row.
841 """
842 return self(row)
844 def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]":
845 """
846 Return all rows for this model.
847 """
848 return self.collect()
850 def get_relationships(self) -> dict[str, Relationship[Any]]:
851 """
852 Return the registered relationships of the current model.
853 """
854 return self._relationships or {}
856 ##########################
857 # TypeDAL Modified Logic #
858 ##########################
860 def insert(self: typing.Type[T_MetaInstance], **fields: Any) -> T_MetaInstance:
861 """
862 This is only called when db.define is not used as a decorator.
864 cls.__table functions as 'self'
866 Args:
867 **fields: anything you want to insert in the database
869 Returns: the ID of the new row.
871 """
872 table = self._ensure_table_defined()
874 result = table.insert(**fields)
875 # it already is an int but mypy doesn't understand that
876 return self(result)
878 def _insert(self, **fields: Any) -> str:
879 table = self._ensure_table_defined()
881 return str(table._insert(**fields))
883 def bulk_insert(self: typing.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRows[T_MetaInstance]":
884 """
885 Insert multiple rows, returns a TypedRows set of new instances.
886 """
887 table = self._ensure_table_defined()
888 result = table.bulk_insert(items)
889 return self.where(lambda row: row.id.belongs(result)).collect()
891 def update_or_insert(
892 self: typing.Type[T_MetaInstance], query: T_Query | AnyDict = DEFAULT, **values: Any
893 ) -> T_MetaInstance:
894 """
895 Update a row if query matches, else insert a new one.
897 Returns the created or updated instance.
898 """
899 table = self._ensure_table_defined()
901 if query is DEFAULT:
902 record = table(**values)
903 elif isinstance(query, dict):
904 record = table(**query)
905 else:
906 record = table(query)
908 if not record:
909 return self.insert(**values)
911 record.update_record(**values)
912 return self(record)
914 def validate_and_insert(
915 self: typing.Type[T_MetaInstance], **fields: Any
916 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:
917 """
918 Validate input data and then insert a row.
920 Returns a tuple of (the created instance, a dict of errors).
921 """
922 table = self._ensure_table_defined()
923 result = table.validate_and_insert(**fields)
924 if row_id := result.get("id"):
925 return self(row_id), None
926 else:
927 return None, result.get("errors")
929 def validate_and_update(
930 self: typing.Type[T_MetaInstance], query: Query, **fields: Any
931 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:
932 """
933 Validate input data and then update max 1 row.
935 Returns a tuple of (the updated instance, a dict of errors).
936 """
937 table = self._ensure_table_defined()
939 result = table.validate_and_update(query, **fields)
941 if errors := result.get("errors"):
942 return None, errors
943 elif row_id := result.get("id"):
944 return self(row_id), None
945 else: # pragma: no cover
946 # update on query without result (shouldnt happen)
947 return None, None
949 def validate_and_update_or_insert(
950 self: typing.Type[T_MetaInstance], query: Query, **fields: Any
951 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:
952 """
953 Validate input data and then update_and_insert (on max 1 row).
955 Returns a tuple of (the updated/created instance, a dict of errors).
956 """
957 table = self._ensure_table_defined()
958 result = table.validate_and_update_or_insert(query, **fields)
960 if errors := result.get("errors"):
961 return None, errors
962 elif row_id := result.get("id"):
963 return self(row_id), None
964 else: # pragma: no cover
965 # update on query without result (shouldnt happen)
966 return None, None
968 def select(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":
969 """
970 See QueryBuilder.select!
971 """
972 return QueryBuilder(self).select(*a, **kw)
974 def paginate(self: typing.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]":
975 """
976 See QueryBuilder.paginate!
977 """
978 return QueryBuilder(self).paginate(limit=limit, page=page)
980 def chunk(
981 self: typing.Type[T_MetaInstance], chunk_size: int
982 ) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:
983 """
984 See QueryBuilder.chunk!
985 """
986 return QueryBuilder(self).chunk(chunk_size)
988 def where(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":
989 """
990 See QueryBuilder.where!
991 """
992 return QueryBuilder(self).where(*a, **kw)
994 def cache(self: typing.Type[T_MetaInstance], *deps: Any, **kwargs: Any) -> "QueryBuilder[T_MetaInstance]":
995 """
996 See QueryBuilder.cache!
997 """
998 return QueryBuilder(self).cache(*deps, **kwargs)
1000 def count(self: typing.Type[T_MetaInstance]) -> int:
1001 """
1002 See QueryBuilder.count!
1003 """
1004 return QueryBuilder(self).count()
1006 def first(self: typing.Type[T_MetaInstance]) -> T_MetaInstance | None:
1007 """
1008 See QueryBuilder.first!
1009 """
1010 return QueryBuilder(self).first()
1012 def first_or_fail(self: typing.Type[T_MetaInstance]) -> T_MetaInstance:
1013 """
1014 See QueryBuilder.first_or_fail!
1015 """
1016 return QueryBuilder(self).first_or_fail()
1018 def join(
1019 self: typing.Type[T_MetaInstance],
1020 *fields: str | typing.Type["TypedTable"],
1021 method: JOIN_OPTIONS = None,
1022 on: OnQuery | list[Expression] | Expression = None,
1023 condition: Condition = None,
1024 ) -> "QueryBuilder[T_MetaInstance]":
1025 """
1026 See QueryBuilder.join!
1027 """
1028 return QueryBuilder(self).join(*fields, on=on, condition=condition, method=method)
1030 def collect(self: typing.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]":
1031 """
1032 See QueryBuilder.collect!
1033 """
1034 return QueryBuilder(self).collect(verbose=verbose)
1036 @property
1037 def ALL(cls) -> pydal.objects.SQLALL:
1038 """
1039 Select all fields for this table.
1040 """
1041 table = cls._ensure_table_defined()
1043 return table.ALL
1045 ##########################
1046 # TypeDAL Shadowed Logic #
1047 ##########################
1048 fields: list[str]
1050 # other table methods:
1052 def truncate(self, mode: str = "") -> None:
1053 """
1054 Remove all data and reset index.
1055 """
1056 table = self._ensure_table_defined()
1057 table.truncate(mode)
1059 def drop(self, mode: str = "") -> None:
1060 """
1061 Remove the underlying table.
1062 """
1063 table = self._ensure_table_defined()
1064 table.drop(mode)
1066 def create_index(self, name: str, *fields: Field | str, **kwargs: Any) -> bool:
1067 """
1068 Add an index on some columns of this table.
1069 """
1070 table = self._ensure_table_defined()
1071 result = table.create_index(name, *fields, **kwargs)
1072 return typing.cast(bool, result)
1074 def drop_index(self, name: str, if_exists: bool = False) -> bool:
1075 """
1076 Remove an index from this table.
1077 """
1078 table = self._ensure_table_defined()
1079 result = table.drop_index(name, if_exists)
1080 return typing.cast(bool, result)
1082 def import_from_csv_file(
1083 self,
1084 csvfile: typing.TextIO,
1085 id_map: dict[str, str] = None,
1086 null: Any = "<NULL>",
1087 unique: str = "uuid",
1088 id_offset: dict[str, int] = None, # id_offset used only when id_map is None
1089 transform: typing.Callable[[dict[Any, Any]], dict[Any, Any]] = None,
1090 validate: bool = False,
1091 encoding: str = "utf-8",
1092 delimiter: str = ",",
1093 quotechar: str = '"',
1094 quoting: int = csv.QUOTE_MINIMAL,
1095 restore: bool = False,
1096 **kwargs: Any,
1097 ) -> None:
1098 """
1099 Load a csv file into the database.
1100 """
1101 table = self._ensure_table_defined()
1102 table.import_from_csv_file(
1103 csvfile,
1104 id_map=id_map,
1105 null=null,
1106 unique=unique,
1107 id_offset=id_offset,
1108 transform=transform,
1109 validate=validate,
1110 encoding=encoding,
1111 delimiter=delimiter,
1112 quotechar=quotechar,
1113 quoting=quoting,
1114 restore=restore,
1115 **kwargs,
1116 )
1118 def on(self, query: Query | bool) -> Expression:
1119 """
1120 Shadow Table.on.
1122 Used for joins.
1124 See Also:
1125 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation
1126 """
1127 table = self._ensure_table_defined()
1128 return typing.cast(Expression, table.on(query))
1130 def with_alias(self, alias: str) -> _Table:
1131 """
1132 Shadow Table.with_alias.
1134 Useful for joins when joining the same table multiple times.
1136 See Also:
1137 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation
1138 """
1139 table = self._ensure_table_defined()
1140 return table.with_alias(alias)
1142 # @typing.dataclass_transform()
1145class TypedField(Expression, typing.Generic[T_Value]): # pragma: no cover
1146 """
1147 Typed version of pydal.Field, which will be converted to a normal Field in the background.
1148 """
1150 # will be set by .bind on db.define
1151 name = ""
1152 _db: Optional[pydal.DAL] = None
1153 _rname: Optional[str] = None
1154 _table: Optional[Table] = None
1155 _field: Optional[Field] = None
1157 _type: T_annotation
1158 kwargs: Any
1160 requires: Validator | typing.Iterable[Validator]
1162 def __init__(self, _type: typing.Type[T_Value] | types.UnionType = str, /, **settings: Any) -> None: # type: ignore
1163 """
1164 A TypedFieldType should not be inited manually, but TypedField (from `fields.py`) should be used!
1165 """
1166 self._type = _type
1167 self.kwargs = settings
1168 # super().__init__()
1170 @typing.overload
1171 def __get__(self, instance: T_MetaInstance, owner: typing.Type[T_MetaInstance]) -> T_Value: # pragma: no cover
1172 """
1173 row.field -> (actual data).
1174 """
1176 @typing.overload
1177 def __get__(self, instance: None, owner: "typing.Type[TypedTable]") -> "TypedField[T_Value]": # pragma: no cover
1178 """
1179 Table.field -> Field.
1180 """
1182 def __get__(
1183 self, instance: T_MetaInstance | None, owner: typing.Type[T_MetaInstance]
1184 ) -> typing.Union[T_Value, "TypedField[T_Value]"]:
1185 """
1186 Since this class is a Descriptor field, \
1187 it returns something else depending on if it's called on a class or instance.
1189 (this is mostly for mypy/typing)
1190 """
1191 if instance:
1192 # this is only reached in a very specific case:
1193 # an instance of the object was created with a specific set of fields selected (excluding the current one)
1194 # in that case, no value was stored in the owner -> return None (since the field was not selected)
1195 return typing.cast(T_Value, None) # cast as T_Value so mypy understands it for selected fields
1196 else:
1197 # getting as class -> return actual field so pydal understands it when using in query etc.
1198 return typing.cast(TypedField[T_Value], self._field) # pretend it's still typed for IDE support
1200 def __str__(self) -> str:
1201 """
1202 String representation of a Typed Field.
1204 If `type` is set explicitly (e.g. TypedField(str, type="text")), that type is used: `TypedField.text`,
1205 otherwise the type annotation is used (e.g. TypedField(str) -> TypedField.str)
1206 """
1207 return str(self._field) if self._field else ""
1209 def __repr__(self) -> str:
1210 """
1211 More detailed string representation of a Typed Field.
1213 Uses __str__ and adds the provided extra options (kwargs) in the representation.
1214 """
1215 s = self.__str__()
1217 if "type" in self.kwargs:
1218 # manual type in kwargs supplied
1219 t = self.kwargs["type"]
1220 elif issubclass(type, type(self._type)):
1221 # normal type, str.__name__ = 'str'
1222 t = getattr(self._type, "__name__", str(self._type))
1223 elif t_args := typing.get_args(self._type):
1224 # list[str] -> 'str'
1225 t = t_args[0].__name__
1226 else: # pragma: no cover
1227 # fallback - something else, may not even happen, I'm not sure
1228 t = self._type
1230 s = f"TypedField[{t}].{s}" if s else f"TypedField[{t}]"
1232 kw = self.kwargs.copy()
1233 kw.pop("type", None)
1234 return f"<{s} with options {kw}>"
1236 def _to_field(self, extra_kwargs: typing.MutableMapping[str, Any]) -> Optional[str]:
1237 """
1238 Convert a Typed Field instance to a pydal.Field.
1239 """
1240 other_kwargs = self.kwargs.copy()
1241 extra_kwargs.update(other_kwargs)
1242 return extra_kwargs.pop("type", False) or TypeDAL._annotation_to_pydal_fieldtype(self._type, extra_kwargs)
1244 def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None:
1245 """
1246 Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`.
1247 """
1248 self._table = table
1249 self._field = field
1251 def __getattr__(self, key: str) -> Any:
1252 """
1253 If the regular getattribute does not work, try to get info from the related Field.
1254 """
1255 with contextlib.suppress(AttributeError):
1256 return super().__getattribute__(key)
1258 # try on actual field:
1259 return getattr(self._field, key)
1261 def __eq__(self, other: Any) -> Query:
1262 """
1263 Performing == on a Field will result in a Query.
1264 """
1265 return typing.cast(Query, self._field == other)
1267 def __ne__(self, other: Any) -> Query:
1268 """
1269 Performing != on a Field will result in a Query.
1270 """
1271 return typing.cast(Query, self._field != other)
1273 def __gt__(self, other: Any) -> Query:
1274 """
1275 Performing > on a Field will result in a Query.
1276 """
1277 return typing.cast(Query, self._field > other)
1279 def __lt__(self, other: Any) -> Query:
1280 """
1281 Performing < on a Field will result in a Query.
1282 """
1283 return typing.cast(Query, self._field < other)
1285 def __ge__(self, other: Any) -> Query:
1286 """
1287 Performing >= on a Field will result in a Query.
1288 """
1289 return typing.cast(Query, self._field >= other)
1291 def __le__(self, other: Any) -> Query:
1292 """
1293 Performing <= on a Field will result in a Query.
1294 """
1295 return typing.cast(Query, self._field <= other)
1297 def __hash__(self) -> int:
1298 """
1299 Shadow Field.__hash__.
1300 """
1301 return hash(self._field)
1303 def __invert__(self) -> Expression:
1304 """
1305 Performing ~ on a Field will result in an Expression.
1306 """
1307 if not self._field: # pragma: no cover
1308 raise ValueError("Unbound Field can not be inverted!")
1310 return typing.cast(Expression, ~self._field)
1312 def lower(self) -> Expression:
1313 """
1314 For string-fields: compare lowercased values.
1315 """
1316 if not self._field: # pragma: no cover
1317 raise ValueError("Unbound Field can not be lowered!")
1319 return typing.cast(Expression, self._field.lower())
1321 # ... etc
1324class _TypedTable:
1325 """
1326 This class is a final shared parent between TypedTable and Mixins.
1328 This needs to exist because otherwise the __on_define__ of Mixins are not executed.
1329 Notably, this class exists at a level ABOVE the `metaclass=TableMeta`,
1330 because otherwise typing gets confused when Mixins are used and multiple types could satisfy
1331 generic 'T subclass of TypedTable'
1332 -> Setting 'TypedTable' as the parent for Mixin does not work at runtime (and works semi at type check time)
1333 """
1335 id: "TypedField[int]"
1337 _before_insert: list[BeforeInsertCallable]
1338 _after_insert: list[AfterInsertCallable]
1339 _before_update: list[BeforeUpdateCallable]
1340 _after_update: list[AfterUpdateCallable]
1341 _before_delete: list[BeforeDeleteCallable]
1342 _after_delete: list[AfterDeleteCallable]
1344 @classmethod
1345 def __on_define__(cls, db: TypeDAL) -> None:
1346 """
1347 Method that can be implemented by tables to do an action after db.define is completed.
1349 This can be useful if you need to add something like requires=IS_NOT_IN_DB(db, "table.field"),
1350 where you need a reference to the current database, which may not exist yet when defining the model.
1351 """
1354class TypedTable(_TypedTable, metaclass=TableMeta):
1355 """
1356 Enhanded modeling system on top of pydal's Table that adds typing and additional functionality.
1357 """
1359 # set up by 'new':
1360 _row: Row | None = None
1362 _with: list[str]
1364 def _setup_instance_methods(self) -> None:
1365 self.as_dict = self._as_dict # type: ignore
1366 self.__json__ = self.as_json = self._as_json # type: ignore
1367 # self.as_yaml = self._as_yaml # type: ignore
1368 self.as_xml = self._as_xml # type: ignore
1370 self.update = self._update # type: ignore
1372 self.delete_record = self._delete_record # type: ignore
1373 self.update_record = self._update_record # type: ignore
1375 def __new__(
1376 cls, row_or_id: typing.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, **filters: Any
1377 ) -> typing.Self:
1378 """
1379 Create a Typed Rows model instance from an existing row, ID or query.
1381 Examples:
1382 MyTable(1)
1383 MyTable(id=1)
1384 MyTable(MyTable.id == 1)
1385 """
1386 table = cls._ensure_table_defined()
1387 inst = super().__new__(cls)
1389 if isinstance(row_or_id, TypedTable):
1390 # existing typed table instance!
1391 return typing.cast(Self, row_or_id)
1393 elif isinstance(row_or_id, pydal.objects.Row):
1394 row = row_or_id
1395 elif row_or_id is not None:
1396 row = table(row_or_id, **filters)
1397 elif filters:
1398 row = table(**filters)
1399 else:
1400 # dummy object
1401 return inst
1403 if not row:
1404 return None # type: ignore
1406 inst._row = row
1407 inst.__dict__.update(row)
1408 inst._setup_instance_methods()
1409 return inst
1411 def __iter__(self) -> typing.Generator[Any, None, None]:
1412 """
1413 Allows looping through the columns.
1414 """
1415 row = self._ensure_matching_row()
1416 yield from iter(row)
1418 def __getitem__(self, item: str) -> Any:
1419 """
1420 Allows dictionary notation to get columns.
1421 """
1422 if item in self.__dict__:
1423 return self.__dict__.get(item)
1425 # fallback to lookup in row
1426 if self._row:
1427 return self._row[item]
1429 # nothing found!
1430 raise KeyError(item)
1432 def __getattr__(self, item: str) -> Any:
1433 """
1434 Allows dot notation to get columns.
1435 """
1436 if value := self.get(item):
1437 return value
1439 raise AttributeError(item)
1441 def get(self, item: str, default: Any = None) -> Any:
1442 """
1443 Try to get a column from this instance, else return default.
1444 """
1445 try:
1446 return self.__getitem__(item)
1447 except KeyError:
1448 return default
1450 def __setitem__(self, key: str, value: Any) -> None:
1451 """
1452 Data can both be updated via dot and dict notation.
1453 """
1454 return setattr(self, key, value)
1456 def __int__(self) -> int:
1457 """
1458 Calling int on a model instance will return its id.
1459 """
1460 return getattr(self, "id", 0)
1462 def __bool__(self) -> bool:
1463 """
1464 If the instance has an underlying row with data, it is truthy.
1465 """
1466 return bool(getattr(self, "_row", False))
1468 def _ensure_matching_row(self) -> Row:
1469 if not getattr(self, "_row", None):
1470 raise EnvironmentError("Trying to access non-existant row. Maybe it was deleted or not yet initialized?")
1471 return self._row
1473 def __repr__(self) -> str:
1474 """
1475 String representation of the model instance.
1476 """
1477 model_name = self.__class__.__name__
1478 model_data = {}
1480 if self._row:
1481 model_data = self._row.as_json()
1483 details = model_name
1484 details += f"({model_data})"
1486 if relationships := getattr(self, "_with", []):
1487 details += f" + {relationships}"
1489 return f"<{details}>"
1491 # serialization
1492 # underscore variants work for class instances (set up by _setup_instance_methods)
1494 @classmethod
1495 def as_dict(cls, flat: bool = False, sanitize: bool = True) -> AnyDict:
1496 """
1497 Dump the object to a plain dict.
1499 Can be used as both a class or instance method:
1500 - dumps the table info if it's a class
1501 - dumps the row info if it's an instance (see _as_dict)
1502 """
1503 table = cls._ensure_table_defined()
1504 result = table.as_dict(flat, sanitize)
1505 return typing.cast(AnyDict, result)
1507 @classmethod
1508 def as_json(cls, sanitize: bool = True, indent: Optional[int] = None, **kwargs: Any) -> str:
1509 """
1510 Dump the object to json.
1512 Can be used as both a class or instance method:
1513 - dumps the table info if it's a class
1514 - dumps the row info if it's an instance (see _as_json)
1515 """
1516 data = cls.as_dict(sanitize=sanitize)
1517 return as_json.encode(data, indent=indent, **kwargs)
1519 @classmethod
1520 def as_xml(cls, sanitize: bool = True) -> str: # pragma: no cover
1521 """
1522 Dump the object to xml.
1524 Can be used as both a class or instance method:
1525 - dumps the table info if it's a class
1526 - dumps the row info if it's an instance (see _as_xml)
1527 """
1528 table = cls._ensure_table_defined()
1529 return typing.cast(str, table.as_xml(sanitize))
1531 @classmethod
1532 def as_yaml(cls, sanitize: bool = True) -> str:
1533 """
1534 Dump the object to yaml.
1536 Can be used as both a class or instance method:
1537 - dumps the table info if it's a class
1538 - dumps the row info if it's an instance (see _as_yaml)
1539 """
1540 table = cls._ensure_table_defined()
1541 return typing.cast(str, table.as_yaml(sanitize))
1543 def _as_dict(
1544 self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None
1545 ) -> AnyDict:
1546 row = self._ensure_matching_row()
1548 result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)
1550 def asdict_method(obj: Any) -> Any: # pragma: no cover
1551 if hasattr(obj, "_as_dict"): # typedal
1552 return obj._as_dict()
1553 elif hasattr(obj, "as_dict"): # pydal
1554 return obj.as_dict()
1555 else: # something else??
1556 return obj.__dict__
1558 if _with := getattr(self, "_with", None):
1559 for relationship in _with:
1560 data = self.get(relationship)
1562 if isinstance(data, list):
1563 data = [asdict_method(_) for _ in data]
1564 elif data:
1565 data = asdict_method(data)
1567 result[relationship] = data
1569 return typing.cast(AnyDict, result)
1571 def _as_json(
1572 self,
1573 default: typing.Callable[[Any], Any] = None,
1574 indent: Optional[int] = None,
1575 **kwargs: Any,
1576 ) -> str:
1577 data = self._as_dict()
1578 return as_json.encode(data, default=default, indent=indent, **kwargs)
1580 def _as_xml(self, sanitize: bool = True) -> str: # pragma: no cover
1581 row = self._ensure_matching_row()
1582 return typing.cast(str, row.as_xml(sanitize))
1584 # def _as_yaml(self, sanitize: bool = True) -> str:
1585 # row = self._ensure_matching_row()
1586 # return typing.cast(str, row.as_yaml(sanitize))
1588 def __setattr__(self, key: str, value: Any) -> None:
1589 """
1590 When setting a property on a Typed Table model instance, also update the underlying row.
1591 """
1592 if self._row and key in self._row.__dict__ and not callable(value):
1593 # enables `row.key = value; row.update_record()`
1594 self._row[key] = value
1596 super().__setattr__(key, value)
1598 @classmethod
1599 def update(cls: typing.Type[T_MetaInstance], query: Query, **fields: Any) -> T_MetaInstance | None:
1600 """
1601 Update one record.
1603 Example:
1604 MyTable.update(MyTable.id == 1, name="NewName") -> MyTable
1605 """
1606 # todo: update multiple?
1607 if record := cls(query):
1608 return record.update_record(**fields)
1609 else:
1610 return None
1612 def _update(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:
1613 row = self._ensure_matching_row()
1614 row.update(**fields)
1615 self.__dict__.update(**fields)
1616 return self
1618 def _update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:
1619 row = self._ensure_matching_row()
1620 new_row = row.update_record(**fields)
1621 self.update(**new_row)
1622 return self
1624 def update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance: # pragma: no cover
1625 """
1626 Here as a placeholder for _update_record.
1628 Will be replaced on instance creation!
1629 """
1630 return self._update_record(**fields)
1632 def _delete_record(self) -> int:
1633 """
1634 Actual logic in `pydal.helpers.classes.RecordDeleter`.
1635 """
1636 row = self._ensure_matching_row()
1637 result = row.delete_record()
1638 self.__dict__ = {} # empty self, since row is no more.
1639 self._row = None # just to be sure
1640 self._setup_instance_methods()
1641 # ^ instance methods might've been deleted by emptying dict,
1642 # but we still want .as_dict to show an error, not the table's as_dict.
1643 return typing.cast(int, result)
1645 def delete_record(self) -> int: # pragma: no cover
1646 """
1647 Here as a placeholder for _delete_record.
1649 Will be replaced on instance creation!
1650 """
1651 return self._delete_record()
1653 # __del__ is also called on the end of a scope so don't remove records on every del!!
1655 # pickling:
1657 def __getstate__(self) -> AnyDict:
1658 """
1659 State to save when pickling.
1661 Prevents db connection from being pickled.
1662 Similar to as_dict but without changing the data of the relationships (dill does that recursively)
1663 """
1664 row = self._ensure_matching_row()
1665 result: AnyDict = row.as_dict()
1667 if _with := getattr(self, "_with", None):
1668 result["_with"] = _with
1669 for relationship in _with:
1670 data = self.get(relationship)
1672 result[relationship] = data
1674 result["_row"] = self._row.as_json() if self._row else ""
1675 return result
1677 def __setstate__(self, state: AnyDict) -> None:
1678 """
1679 Used by dill when loading from a bytestring.
1680 """
1681 # as_dict also includes table info, so dump as json to only get the actual row data
1682 # then create a new (more empty) row object:
1683 state["_row"] = Row(json.loads(state["_row"]))
1684 self.__dict__ |= state
1687# backwards compat:
1688TypedRow = TypedTable
1691class TypedRows(typing.Collection[T_MetaInstance], Rows):
1692 """
1693 Slighly enhaned and typed functionality on top of pydal Rows (the result of a select).
1694 """
1696 records: dict[int, T_MetaInstance]
1697 # _rows: Rows
1698 model: typing.Type[T_MetaInstance]
1699 metadata: Metadata
1701 # pseudo-properties: actually stored in _rows
1702 db: TypeDAL
1703 colnames: list[str]
1704 fields: list[Field]
1705 colnames_fields: list[Field]
1706 response: list[tuple[Any, ...]]
1708 def __init__(
1709 self,
1710 rows: Rows,
1711 model: typing.Type[T_MetaInstance],
1712 records: dict[int, T_MetaInstance] = None,
1713 metadata: Metadata = None,
1714 ) -> None:
1715 """
1716 Should not be called manually!
1718 Normally, the `records` from an existing `Rows` object are used
1719 but these can be overwritten with a `records` dict.
1720 `metadata` can be any (un)structured data
1721 `model` is a Typed Table class
1722 """
1723 records = records or {row.id: model(row) for row in rows}
1724 super().__init__(rows.db, records, rows.colnames, rows.compact, rows.response, rows.fields)
1725 self.model = model
1726 self.metadata = metadata or {}
1727 self.colnames = rows.colnames
1729 def __len__(self) -> int:
1730 """
1731 Return the count of rows.
1732 """
1733 return len(self.records)
1735 def __iter__(self) -> typing.Iterator[T_MetaInstance]:
1736 """
1737 Loop through the rows.
1738 """
1739 yield from self.records.values()
1741 def __contains__(self, ind: Any) -> bool:
1742 """
1743 Check if an id exists in this result set.
1744 """
1745 return ind in self.records
1747 def first(self) -> T_MetaInstance | None:
1748 """
1749 Get the row with the lowest id.
1750 """
1751 if not self.records:
1752 return None
1754 return next(iter(self))
1756 def last(self) -> T_MetaInstance | None:
1757 """
1758 Get the row with the highest id.
1759 """
1760 if not self.records:
1761 return None
1763 max_id = max(self.records.keys())
1764 return self[max_id]
1766 def find(
1767 self, f: typing.Callable[[T_MetaInstance], Query], limitby: tuple[int, int] = None
1768 ) -> "TypedRows[T_MetaInstance]":
1769 """
1770 Returns a new Rows object, a subset of the original object, filtered by the function `f`.
1771 """
1772 if not self.records:
1773 return self.__class__(self, self.model, {})
1775 records = {}
1776 if limitby:
1777 _min, _max = limitby
1778 else:
1779 _min, _max = 0, len(self)
1780 count = 0
1781 for i, row in self.records.items():
1782 if f(row):
1783 if _min <= count:
1784 records[i] = row
1785 count += 1
1786 if count == _max:
1787 break
1789 return self.__class__(self, self.model, records)
1791 def exclude(self, f: typing.Callable[[T_MetaInstance], Query]) -> "TypedRows[T_MetaInstance]":
1792 """
1793 Removes elements from the calling Rows object, filtered by the function `f`, \
1794 and returns a new Rows object containing the removed elements.
1795 """
1796 if not self.records:
1797 return self.__class__(self, self.model, {})
1798 removed = {}
1799 to_remove = []
1800 for i in self.records:
1801 row = self[i]
1802 if f(row):
1803 removed[i] = self.records[i]
1804 to_remove.append(i)
1806 [self.records.pop(i) for i in to_remove]
1808 return self.__class__(
1809 self,
1810 self.model,
1811 removed,
1812 )
1814 def sort(self, f: typing.Callable[[T_MetaInstance], Any], reverse: bool = False) -> list[T_MetaInstance]:
1815 """
1816 Returns a list of sorted elements (not sorted in place).
1817 """
1818 return [r for (r, s) in sorted(zip(self.records.values(), self), key=lambda r: f(r[1]), reverse=reverse)]
1820 def __str__(self) -> str:
1821 """
1822 Simple string representation.
1823 """
1824 return f"<TypedRows with {len(self)} records>"
1826 def __repr__(self) -> str:
1827 """
1828 Print a table on repr().
1829 """
1830 data = self.as_dict()
1831 headers = list(next(iter(data.values())).keys())
1832 return mktable(data, headers)
1834 def group_by_value(
1835 self, *fields: "str | Field | TypedField[T]", one_result: bool = False, **kwargs: Any
1836 ) -> dict[T, list[T_MetaInstance]]:
1837 """
1838 Group the rows by a specific field (which will be the dict key).
1839 """
1840 kwargs["one_result"] = one_result
1841 result = super().group_by_value(*fields, **kwargs)
1842 return typing.cast(dict[T, list[T_MetaInstance]], result)
1844 def column(self, column: str = None) -> list[Any]:
1845 """
1846 Get a list of all values in a specific column.
1848 Example:
1849 rows.column('name') -> ['Name 1', 'Name 2', ...]
1850 """
1851 return typing.cast(list[Any], super().column(column))
1853 def as_csv(self) -> str:
1854 """
1855 Dump the data to csv.
1856 """
1857 return typing.cast(str, super().as_csv())
1859 def as_dict(
1860 self,
1861 key: str = None,
1862 compact: bool = False,
1863 storage_to_dict: bool = False,
1864 datetime_to_str: bool = False,
1865 custom_types: list[type] = None,
1866 ) -> dict[int, AnyDict]:
1867 """
1868 Get the data in a dict of dicts.
1869 """
1870 if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):
1871 # functionality not guaranteed
1872 return typing.cast(
1873 dict[int, AnyDict],
1874 super().as_dict(
1875 key or "id",
1876 compact,
1877 storage_to_dict,
1878 datetime_to_str,
1879 custom_types,
1880 ),
1881 )
1883 return {k: v.as_dict() for k, v in self.records.items()}
1885 def as_json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:
1886 """
1887 Turn the data into a dict and then dump to JSON.
1888 """
1889 data = self.as_list()
1891 return as_json.encode(data, default=default, indent=indent, **kwargs)
1893 def json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:
1894 """
1895 Turn the data into a dict and then dump to JSON.
1896 """
1897 return self.as_json(default=default, indent=indent, **kwargs)
1899 def as_list(
1900 self,
1901 compact: bool = False,
1902 storage_to_dict: bool = False,
1903 datetime_to_str: bool = False,
1904 custom_types: list[type] = None,
1905 ) -> list[AnyDict]:
1906 """
1907 Get the data in a list of dicts.
1908 """
1909 if any([compact, storage_to_dict, datetime_to_str, custom_types]):
1910 return typing.cast(list[AnyDict], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types))
1912 return [_.as_dict() for _ in self.records.values()]
1914 def __getitem__(self, item: int) -> T_MetaInstance:
1915 """
1916 You can get a specific row by ID from a typedrows by using rows[idx] notation.
1918 Since pydal's implementation differs (they expect a list instead of a dict with id keys),
1919 using rows[0] will return the first row, regardless of its id.
1920 """
1921 try:
1922 return self.records[item]
1923 except KeyError as e:
1924 if item == 0 and (row := self.first()):
1925 # special case: pydal internals think Rows.records is a list, not a dict
1926 return row
1928 raise e
1930 def get(self, item: int) -> typing.Optional[T_MetaInstance]:
1931 """
1932 Get a row by ID, or receive None if it isn't in this result set.
1933 """
1934 return self.records.get(item)
1936 def update(self, **new_values: Any) -> bool:
1937 """
1938 Update the current rows in the database with new_values.
1939 """
1940 # cast to make mypy understand .id is a TypedField and not an int!
1941 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())
1943 ids = set(self.column("id"))
1944 query = table.id.belongs(ids)
1945 return bool(self.db(query).update(**new_values))
1947 def delete(self) -> bool:
1948 """
1949 Delete the currently selected rows from the database.
1950 """
1951 # cast to make mypy understand .id is a TypedField and not an int!
1952 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())
1954 ids = set(self.column("id"))
1955 query = table.id.belongs(ids)
1956 return bool(self.db(query).delete())
1958 def join(
1959 self,
1960 field: "Field | TypedField[Any]",
1961 name: str = None,
1962 constraint: Query = None,
1963 fields: list[str | Field] = None,
1964 orderby: Optional[str | Field] = None,
1965 ) -> T_MetaInstance:
1966 """
1967 This can be used to JOIN with some relationships after the initial select.
1969 Using the querybuilder's .join() method is prefered!
1970 """
1971 result = super().join(field, name, constraint, fields or [], orderby)
1972 return typing.cast(T_MetaInstance, result)
1974 def export_to_csv_file(
1975 self,
1976 ofile: typing.TextIO,
1977 null: Any = "<NULL>",
1978 delimiter: str = ",",
1979 quotechar: str = '"',
1980 quoting: int = csv.QUOTE_MINIMAL,
1981 represent: bool = False,
1982 colnames: list[str] = None,
1983 write_colnames: bool = True,
1984 *args: Any,
1985 **kwargs: Any,
1986 ) -> None:
1987 """
1988 Shadow export_to_csv_file from Rows, but with typing.
1990 See http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#Exporting-and-importing-data
1991 """
1992 super().export_to_csv_file(
1993 ofile,
1994 null,
1995 *args,
1996 delimiter=delimiter,
1997 quotechar=quotechar,
1998 quoting=quoting,
1999 represent=represent,
2000 colnames=colnames or self.colnames,
2001 write_colnames=write_colnames,
2002 **kwargs,
2003 )
2005 @classmethod
2006 def from_rows(
2007 cls, rows: Rows, model: typing.Type[T_MetaInstance], metadata: Metadata = None
2008 ) -> "TypedRows[T_MetaInstance]":
2009 """
2010 Internal method to convert a Rows object to a TypedRows.
2011 """
2012 return cls(rows, model, metadata=metadata)
2014 def __getstate__(self) -> AnyDict:
2015 """
2016 Used by dill to dump to bytes (exclude db connection etc).
2017 """
2018 return {
2019 "metadata": json.dumps(self.metadata, default=str),
2020 "records": self.records,
2021 "model": str(self.model._table),
2022 "colnames": self.colnames,
2023 }
2025 def __setstate__(self, state: AnyDict) -> None:
2026 """
2027 Used by dill when loading from a bytestring.
2028 """
2029 state["metadata"] = json.loads(state["metadata"])
2030 self.__dict__.update(state)
2031 # db etc. set after undill by caching.py
2034from .caching import ( # noqa: E402
2035 _remove_cache,
2036 _TypedalCache,
2037 _TypedalCacheDependency,
2038 create_and_hash_cache_key,
2039 get_expire,
2040 load_from_cache,
2041 save_to_cache,
2042)
2045class QueryBuilder(typing.Generic[T_MetaInstance]):
2046 """
2047 Abstration on top of pydal's query system.
2048 """
2050 model: typing.Type[T_MetaInstance]
2051 query: Query
2052 select_args: list[Any]
2053 select_kwargs: SelectKwargs
2054 relationships: dict[str, Relationship[Any]]
2055 metadata: Metadata
2057 def __init__(
2058 self,
2059 model: typing.Type[T_MetaInstance],
2060 add_query: Optional[Query] = None,
2061 select_args: Optional[list[Any]] = None,
2062 select_kwargs: Optional[SelectKwargs] = None,
2063 relationships: dict[str, Relationship[Any]] = None,
2064 metadata: Metadata = None,
2065 ):
2066 """
2067 Normally, you wouldn't manually initialize a QueryBuilder but start using a method on a TypedTable.
2069 Example:
2070 MyTable.where(...) -> QueryBuilder[MyTable]
2071 """
2072 self.model = model
2073 table = model._ensure_table_defined()
2074 default_query = typing.cast(Query, table.id > 0)
2075 self.query = add_query or default_query
2076 self.select_args = select_args or []
2077 self.select_kwargs = select_kwargs or {}
2078 self.relationships = relationships or {}
2079 self.metadata = metadata or {}
2081 def __str__(self) -> str:
2082 """
2083 Simple string representation for the query builder.
2084 """
2085 return f"QueryBuilder for {self.model}"
2087 def __repr__(self) -> str:
2088 """
2089 Advanced string representation for the query builder.
2090 """
2091 return (
2092 f"<QueryBuilder for {self.model} with "
2093 f"{len(self.select_args)} select args; "
2094 f"{len(self.select_kwargs)} select kwargs; "
2095 f"{len(self.relationships)} relationships; "
2096 f"query: {bool(self.query)}; "
2097 f"metadata: {self.metadata}; "
2098 f">"
2099 )
2101 def __bool__(self) -> bool:
2102 """
2103 Querybuilder is truthy if it has rows.
2104 """
2105 return self.count() > 0
2107 def _extend(
2108 self,
2109 add_query: Optional[Query] = None,
2110 overwrite_query: Optional[Query] = None,
2111 select_args: Optional[list[Any]] = None,
2112 select_kwargs: Optional[SelectKwargs] = None,
2113 relationships: dict[str, Relationship[Any]] = None,
2114 metadata: Metadata = None,
2115 ) -> "QueryBuilder[T_MetaInstance]":
2116 return QueryBuilder(
2117 self.model,
2118 (add_query & self.query) if add_query else overwrite_query or self.query,
2119 (self.select_args + select_args) if select_args else self.select_args,
2120 (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs,
2121 (self.relationships | relationships) if relationships else self.relationships,
2122 (self.metadata | (metadata or {})) if metadata else self.metadata,
2123 )
2125 def select(self, *fields: Any, **options: Unpack[SelectKwargs]) -> "QueryBuilder[T_MetaInstance]":
2126 """
2127 Fields: database columns by name ('id'), by field reference (table.id) or other (e.g. table.ALL).
2129 Options:
2130 paraphrased from the web2py pydal docs,
2131 For more info, see http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache
2133 orderby: field(s) to order by. Supported:
2134 table.name - sort by name, ascending
2135 ~table.name - sort by name, descending
2136 <random> - sort randomly
2137 table.name|table.id - sort by two fields (first name, then id)
2139 groupby, having: together with orderby:
2140 groupby can be a field (e.g. table.name) to group records by
2141 having can be a query, only those `having` the condition are grouped
2143 limitby: tuple of min and max. When using the query builder, .paginate(limit, page) is recommended.
2144 distinct: bool/field. Only select rows that differ
2145 orderby_on_limitby (bool, default: True): by default, an implicit orderby is added when doing limitby.
2146 join: othertable.on(query) - do an INNER JOIN. Using TypeDAL relationships with .join() is recommended!
2147 left: othertable.on(query) - do a LEFT JOIN. Using TypeDAL relationships with .join() is recommended!
2148 cache: cache the query result to speed up repeated queries; e.g. (cache=(cache.ram, 3600), cacheable=True)
2149 """
2150 return self._extend(select_args=list(fields), select_kwargs=options)
2152 def where(
2153 self,
2154 *queries_or_lambdas: Query | typing.Callable[[typing.Type[T_MetaInstance]], Query],
2155 **filters: Any,
2156 ) -> "QueryBuilder[T_MetaInstance]":
2157 """
2158 Extend the builder's query.
2160 Can be used in multiple ways:
2161 .where(Query) -> with a direct query such as `Table.id == 5`
2162 .where(lambda table: table.id == 5) -> with a query via a lambda
2163 .where(id=5) -> via keyword arguments
2165 When using multiple where's, they will be ANDed:
2166 .where(lambda table: table.id == 5).where(lambda table: table.id == 6) == (table.id == 5) & (table.id=6)
2167 When passing multiple queries to a single .where, they will be ORed:
2168 .where(lambda table: table.id == 5, lambda table: table.id == 6) == (table.id == 5) | (table.id=6)
2169 """
2170 new_query = self.query
2171 table = self.model._ensure_table_defined()
2173 for field, value in filters.items():
2174 new_query &= table[field] == value
2176 subquery: DummyQuery | Query = DummyQuery()
2177 for query_or_lambda in queries_or_lambdas:
2178 if isinstance(query_or_lambda, _Query):
2179 subquery |= typing.cast(Query, query_or_lambda)
2180 elif callable(query_or_lambda):
2181 if result := query_or_lambda(self.model):
2182 subquery |= result
2183 elif isinstance(query_or_lambda, (Field, _Field)) or is_typed_field(query_or_lambda):
2184 subquery |= typing.cast(Query, query_or_lambda != None)
2185 else:
2186 raise ValueError(f"Unexpected query type ({type(query_or_lambda)}).")
2188 if subquery:
2189 new_query &= subquery
2191 return self._extend(overwrite_query=new_query)
2193 def join(
2194 self,
2195 *fields: str | typing.Type[TypedTable],
2196 method: JOIN_OPTIONS = None,
2197 on: OnQuery | list[Expression] | Expression = None,
2198 condition: Condition = None,
2199 ) -> "QueryBuilder[T_MetaInstance]":
2200 """
2201 Include relationship fields in the result.
2203 `fields` can be names of Relationships on the current model.
2204 If no fields are passed, all will be used.
2206 By default, the `method` defined in the relationship is used.
2207 This can be overwritten with the `method` keyword argument (left or inner)
2208 """
2209 # todo: allow limiting amount of related rows returned for join?
2211 relationships = self.model.get_relationships()
2213 if condition and on:
2214 raise ValueError("condition and on can not be used together!")
2215 elif condition:
2216 if len(fields) != 1:
2217 raise ValueError("join(field, condition=...) can only be used with exactly one field!")
2219 if isinstance(condition, pydal.objects.Query):
2220 condition = as_lambda(condition)
2222 relationships = {str(fields[0]): relationship(fields[0], condition=condition, join=method)}
2223 elif on:
2224 if len(fields) != 1:
2225 raise ValueError("join(field, on=...) can only be used with exactly one field!")
2227 if isinstance(on, pydal.objects.Expression):
2228 on = [on]
2230 if isinstance(on, list):
2231 on = as_lambda(on)
2232 relationships = {str(fields[0]): relationship(fields[0], on=on, join=method)}
2234 else:
2235 if fields:
2236 # join on every relationship
2237 relationships = {str(k): relationships[str(k)] for k in fields}
2239 if method:
2240 relationships = {str(k): r.clone(join=method) for k, r in relationships.items()}
2242 return self._extend(relationships=relationships)
2244 def cache(
2245 self, *deps: Any, expires_at: Optional[dt.datetime] = None, ttl: Optional[int | dt.timedelta] = None
2246 ) -> "QueryBuilder[T_MetaInstance]":
2247 """
2248 Enable caching for this query to load repeated calls from a dill row \
2249 instead of executing the sql and collecing matching rows again.
2250 """
2251 existing = self.metadata.get("cache", {})
2253 metadata: Metadata = {}
2255 cache_meta = typing.cast(
2256 CacheMetadata,
2257 self.metadata.get("cache", {})
2258 | {
2259 "enabled": True,
2260 "depends_on": existing.get("depends_on", []) + [str(_) for _ in deps],
2261 "expires_at": get_expire(expires_at=expires_at, ttl=ttl),
2262 },
2263 )
2265 metadata["cache"] = cache_meta
2266 return self._extend(metadata=metadata)
2268 def _get_db(self) -> TypeDAL:
2269 if db := self.model._db:
2270 return db
2271 else: # pragma: no cover
2272 raise EnvironmentError("@define or db.define is not called on this class yet!")
2274 def _select_arg_convert(self, arg: Any) -> Any:
2275 # typedfield are not really used at runtime anymore, but leave it in for safety:
2276 if isinstance(arg, TypedField): # pragma: no cover
2277 arg = arg._field
2279 return arg
2281 def delete(self) -> list[int]:
2282 """
2283 Based on the current query, delete rows and return a list of deleted IDs.
2284 """
2285 db = self._get_db()
2286 removed_ids = [_.id for _ in db(self.query).select("id")]
2287 if db(self.query).delete():
2288 # success!
2289 return removed_ids
2291 return []
2293 def _delete(self) -> str:
2294 db = self._get_db()
2295 return str(db(self.query)._delete())
2297 def update(self, **fields: Any) -> list[int]:
2298 """
2299 Based on the current query, update `fields` and return a list of updated IDs.
2300 """
2301 # todo: limit?
2302 db = self._get_db()
2303 updated_ids = db(self.query).select("id").column("id")
2304 if db(self.query).update(**fields):
2305 # success!
2306 return updated_ids
2308 return []
2310 def _update(self, **fields: Any) -> str:
2311 db = self._get_db()
2312 return str(db(self.query)._update(**fields))
2314 def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], SelectKwargs]:
2315 select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]
2316 select_kwargs = self.select_kwargs.copy()
2317 query = self.query
2318 model = self.model
2319 mut_metadata["query"] = query
2320 # require at least id of main table:
2321 select_fields = ", ".join([str(_) for _ in select_args])
2322 tablename = str(model)
2324 if add_id and f"{tablename}.id" not in select_fields:
2325 # fields of other selected, but required ID is missing.
2326 select_args.append(model.id)
2328 if self.relationships:
2329 query, select_args = self._handle_relationships_pre_select(query, select_args, select_kwargs, mut_metadata)
2331 return query, select_args, select_kwargs
2333 def to_sql(self, add_id: bool = False) -> str:
2334 """
2335 Generate the SQL for the built query.
2336 """
2337 db = self._get_db()
2339 query, select_args, select_kwargs = self._before_query({}, add_id=add_id)
2341 return str(db(query)._select(*select_args, **select_kwargs))
2343 def _collect(self) -> str:
2344 """
2345 Alias for to_sql, pydal-like syntax.
2346 """
2347 return self.to_sql()
2349 def _collect_cached(self, metadata: Metadata) -> "TypedRows[T_MetaInstance] | None":
2350 expires_at = metadata["cache"].get("expires_at")
2351 metadata["cache"] |= {
2352 # key is partly dependant on cache metadata but not these:
2353 "key": None,
2354 "status": None,
2355 "cached_at": None,
2356 "expires_at": None,
2357 }
2359 _, key = create_and_hash_cache_key(
2360 self.model,
2361 metadata,
2362 self.query,
2363 self.select_args,
2364 self.select_kwargs,
2365 self.relationships.keys(),
2366 )
2368 # re-set after creating key:
2369 metadata["cache"]["expires_at"] = expires_at
2370 metadata["cache"]["key"] = key
2372 return load_from_cache(key, self._get_db())
2374 def execute(self, add_id: bool = False) -> Rows:
2375 """
2376 Raw version of .collect which only executes the SQL, without performing any magic afterwards.
2377 """
2378 db = self._get_db()
2379 metadata = typing.cast(Metadata, self.metadata.copy())
2381 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)
2383 return db(query).select(*select_args, **select_kwargs)
2385 def collect(
2386 self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True
2387 ) -> "TypedRows[T_MetaInstance]":
2388 """
2389 Execute the built query and turn it into model instances, while handling relationships.
2390 """
2391 if _to is None:
2392 _to = TypedRows
2394 db = self._get_db()
2395 metadata = typing.cast(Metadata, self.metadata.copy())
2397 if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)):
2398 return result
2400 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)
2402 metadata["sql"] = db(query)._select(*select_args, **select_kwargs)
2404 if verbose: # pragma: no cover
2405 print(metadata["sql"])
2407 rows: Rows = db(query).select(*select_args, **select_kwargs)
2409 metadata["final_query"] = str(query)
2410 metadata["final_args"] = [str(_) for _ in select_args]
2411 metadata["final_kwargs"] = select_kwargs
2413 if verbose: # pragma: no cover
2414 print(rows)
2416 if not self.relationships:
2417 # easy
2418 typed_rows = _to.from_rows(rows, self.model, metadata=metadata)
2420 else:
2421 # harder: try to match rows to the belonging objects
2422 # assume structure of {'table': <data>} per row.
2423 # if that's not the case, return default behavior again
2424 typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to)
2426 # only saves if requested in metadata:
2427 return save_to_cache(typed_rows, rows)
2429 def _handle_relationships_pre_select(
2430 self,
2431 query: Query,
2432 select_args: list[Any],
2433 select_kwargs: SelectKwargs,
2434 metadata: Metadata,
2435 ) -> tuple[Query, list[Any]]:
2436 db = self._get_db()
2437 model = self.model
2439 metadata["relationships"] = set(self.relationships.keys())
2441 # query = self._update_query_for_inner(db, model, query)
2442 join = []
2443 for key, relation in self.relationships.items():
2444 if not relation.condition or relation.join != "inner":
2445 continue
2447 other = relation.get_table(db)
2448 other = other.with_alias(f"{key}_{hash(relation)}")
2449 join.append(other.on(relation.condition(model, other)))
2451 if limitby := select_kwargs.pop("limitby", ()):
2453 # if limitby + relationships:
2454 # 1. get IDs of main table entries that match 'query'
2455 # 2. change query to .belongs(id)
2456 # 3. add joins etc
2458 kwargs: SelectKwargs = select_kwargs | {"limitby": limitby}
2459 # if orderby := select_kwargs.get("orderby"):
2460 # kwargs["orderby"] = orderby
2462 if join:
2463 kwargs["join"] = join
2465 ids = db(query)._select(model.id, **kwargs)
2466 query = model.id.belongs(ids)
2467 metadata["ids"] = ids
2469 if join:
2470 select_kwargs["join"] = join
2472 left = []
2474 for key, relation in self.relationships.items():
2475 other = relation.get_table(db)
2476 method: JOIN_OPTIONS = relation.join or DEFAULT_JOIN_OPTION
2478 select_fields = ", ".join([str(_) for _ in select_args])
2479 pre_alias = str(other)
2481 if f"{other}." not in select_fields:
2482 # no fields of other selected. add .ALL:
2483 select_args.append(other.ALL)
2484 elif f"{other}.id" not in select_fields:
2485 # fields of other selected, but required ID is missing.
2486 select_args.append(other.id)
2488 if relation.on:
2489 # if it has a .on, it's always a left join!
2490 on = relation.on(model, other)
2491 if not isinstance(on, list): # pragma: no cover
2492 on = [on]
2494 left.extend(on)
2495 elif method == "left":
2496 # .on not given, generate it:
2497 other = other.with_alias(f"{key}_{hash(relation)}")
2498 condition = typing.cast(Query, relation.condition(model, other))
2499 left.append(other.on(condition))
2500 else:
2501 # else: inner join (handled earlier)
2502 other = other.with_alias(f"{key}_{hash(relation)}") # only for replace
2503 # other = other.with_alias(f"{key}_{hash(relation)}")
2504 # query &= relation.condition(model, other)
2506 # if no fields of 'other' are included, add other.ALL
2507 # else: only add other.id if missing
2508 select_fields = ", ".join([str(_) for _ in select_args])
2510 post_alias = str(other).split(" AS ")[-1]
2511 if pre_alias != post_alias:
2512 # replace .select's with aliased:
2513 select_fields = select_fields.replace(
2514 f"{pre_alias}.",
2515 f"{post_alias}.",
2516 )
2518 select_args = select_fields.split(", ")
2520 select_kwargs["left"] = left
2521 return query, select_args
2523 def _collect_with_relationships(
2524 self, rows: Rows, metadata: Metadata, _to: typing.Type["TypedRows[Any]"]
2525 ) -> "TypedRows[T_MetaInstance]":
2526 """
2527 Transform the raw rows into Typed Table model instances.
2528 """
2529 db = self._get_db()
2530 main_table = self.model._ensure_table_defined()
2532 records = {}
2533 seen_relations: dict[str, set[str]] = defaultdict(set) # main id -> set of col + id for relation
2535 for row in rows:
2536 main = row[main_table]
2537 main_id = main.id
2539 if main_id not in records:
2540 records[main_id] = self.model(main)
2541 records[main_id]._with = list(self.relationships.keys())
2543 # setup up all relationship defaults (once)
2544 for col, relationship in self.relationships.items():
2545 records[main_id][col] = [] if relationship.multiple else None
2547 # now add other relationship data
2548 for column, relation in self.relationships.items():
2549 relationship_column = f"{column}_{hash(relation)}"
2551 # relationship_column works for aliases with the same target column.
2552 # if col + relationship not in the row, just use the regular name.
2554 relation_data = (
2555 row[relationship_column] if relationship_column in row else row[relation.get_table_name()]
2556 )
2558 if relation_data.id is None:
2559 # always skip None ids
2560 continue
2562 if f"{column}-{relation_data.id}" in seen_relations[main_id]:
2563 # speed up duplicates
2564 continue
2565 else:
2566 seen_relations[main_id].add(f"{column}-{relation_data.id}")
2568 relation_table = relation.get_table(db)
2569 # hopefully an instance of a typed table and a regular row otherwise:
2570 instance = relation_table(relation_data) if looks_like(relation_table, TypedTable) else relation_data
2572 if relation.multiple:
2573 # create list of T
2574 if not isinstance(records[main_id].get(column), list): # pragma: no cover
2575 # should already be set up before!
2576 setattr(records[main_id], column, [])
2578 records[main_id][column].append(instance)
2579 else:
2580 # create single T
2581 records[main_id][column] = instance
2583 return _to(rows, self.model, records, metadata=metadata)
2585 def collect_or_fail(self, exception: Exception = None) -> "TypedRows[T_MetaInstance]":
2586 """
2587 Call .collect() and raise an error if nothing found.
2589 Basically unwraps Optional type.
2590 """
2591 if result := self.collect():
2592 return result
2594 if not exception:
2595 exception = ValueError("Nothing found!")
2597 raise exception
2599 def __iter__(self) -> typing.Generator[T_MetaInstance, None, None]:
2600 """
2601 You can start iterating a Query Builder object before calling collect, for ease of use.
2602 """
2603 yield from self.collect()
2605 def count(self) -> int:
2606 """
2607 Return the amount of rows matching the current query.
2608 """
2609 db = self._get_db()
2610 model = self.model
2611 query = self.query
2613 for key, relation in self.relationships.items():
2614 if not relation.condition or relation.join != "inner":
2615 continue
2617 other = relation.get_table(db)
2618 other = other.with_alias(f"{key}_{hash(relation)}")
2619 query &= relation.condition(model, other)
2621 return db(query).count()
2623 def __paginate(
2624 self,
2625 limit: int,
2626 page: int = 1,
2627 ) -> "QueryBuilder[T_MetaInstance]":
2628 _from = limit * (page - 1)
2629 _to = limit * page
2631 available = self.count()
2633 metadata: Metadata = {}
2635 metadata["pagination"] = {
2636 "limit": limit,
2637 "current_page": page,
2638 "max_page": math.ceil(available / limit),
2639 "rows": available,
2640 "min_max": (_from, _to),
2641 }
2643 return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata)
2645 def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]":
2646 """
2647 Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset.
2649 Note: when using relationships, this limit is only applied to the 'main' table and any number of extra rows \
2650 can be loaded with relationship data!
2651 """
2652 builder = self.__paginate(limit, page)
2654 rows = typing.cast(PaginatedRows[T_MetaInstance], builder.collect(verbose=verbose, _to=PaginatedRows))
2656 rows._query_builder = builder
2657 return rows
2659 def _paginate(
2660 self,
2661 limit: int,
2662 page: int = 1,
2663 ) -> str:
2664 builder = self.__paginate(limit, page)
2665 return builder._collect()
2667 def chunk(self, chunk_size: int) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:
2668 """
2669 Generator that yields rows from a paginated source in chunks.
2671 This function retrieves rows from a paginated data source in chunks of the
2672 specified `chunk_size` and yields them as TypedRows.
2674 Example:
2675 ```
2676 for chunk_of_rows in Table.where(SomeTable.id > 5).chunk(100):
2677 for row in chunk_of_rows:
2678 # Process each row within the chunk.
2679 pass
2680 ```
2681 """
2682 page = 1
2684 while rows := self.__paginate(chunk_size, page).collect():
2685 yield rows
2686 page += 1
2688 def first(self, verbose: bool = False) -> T_MetaInstance | None:
2689 """
2690 Get the first row matching the currently built query.
2692 Also adds paginate, since it would be a waste to select more rows than needed.
2693 """
2694 if row := self.paginate(page=1, limit=1, verbose=verbose).first():
2695 return self.model.from_row(row)
2696 else:
2697 return None
2699 def _first(self) -> str:
2700 return self._paginate(page=1, limit=1)
2702 def first_or_fail(self, exception: Exception = None, verbose: bool = False) -> T_MetaInstance:
2703 """
2704 Call .first() and raise an error if nothing found.
2706 Basically unwraps Optional type.
2707 """
2708 if inst := self.first(verbose=verbose):
2709 return inst
2711 if not exception:
2712 exception = ValueError("Nothing found!")
2714 raise exception
2717S = typing.TypeVar("S")
2720class PaginatedRows(TypedRows[T_MetaInstance]):
2721 """
2722 Extension on top of rows that is used when calling .paginate() instead of .collect().
2723 """
2725 _query_builder: QueryBuilder[T_MetaInstance]
2727 @property
2728 def data(self) -> list[T_MetaInstance]:
2729 """
2730 Get the underlying data.
2731 """
2732 return list(self.records.values())
2734 @property
2735 def pagination(self) -> Pagination:
2736 """
2737 Get all page info.
2738 """
2739 pagination_data = self.metadata["pagination"]
2741 has_next_page = pagination_data["current_page"] < pagination_data["max_page"]
2742 has_prev_page = pagination_data["current_page"] > 1
2743 return {
2744 "total_items": pagination_data["rows"],
2745 "current_page": pagination_data["current_page"],
2746 "per_page": pagination_data["limit"],
2747 "total_pages": pagination_data["max_page"],
2748 "has_next_page": has_next_page,
2749 "has_prev_page": has_prev_page,
2750 "next_page": pagination_data["current_page"] + 1 if has_next_page else None,
2751 "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,
2752 }
2754 def next(self) -> Self:
2755 """
2756 Get the next page.
2757 """
2758 data = self.metadata["pagination"]
2759 if data["current_page"] >= data["max_page"]:
2760 raise StopIteration("Final Page")
2762 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)
2764 def previous(self) -> Self:
2765 """
2766 Get the previous page.
2767 """
2768 data = self.metadata["pagination"]
2769 if data["current_page"] <= 1:
2770 raise StopIteration("First Page")
2772 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)
2774 def as_dict(self, *_: Any, **__: Any) -> PaginateDict: # type: ignore
2775 """
2776 Convert to a dictionary with pagination info and original data.
2778 All arguments are ignored!
2779 """
2780 return {"data": super().as_dict(), "pagination": self.pagination}
2783class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover
2784 """
2785 Used to make pydal Set more typed.
2787 This class is not actually used, only 'cast' by TypeDAL.__call__
2788 """
2790 def count(self, distinct: bool = None, cache: AnyDict = None) -> int:
2791 """
2792 Count returns an int.
2793 """
2794 result = super().count(distinct, cache)
2795 return typing.cast(int, result)
2797 def select(self, *fields: Any, **attributes: Any) -> TypedRows[T_MetaInstance]:
2798 """
2799 Select returns a TypedRows of a user defined table.
2801 Example:
2802 result: TypedRows[MyTable] = db(MyTable.id > 0).select()
2804 for row in result:
2805 typing.reveal_type(row) # MyTable
2806 """
2807 rows = super().select(*fields, **attributes)
2808 return typing.cast(TypedRows[T_MetaInstance], rows)