Coverage for src/typedal/core.py: 100%
904 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-08 16:37 +0200
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-08 16:37 +0200
1"""
2Core functionality of TypeDAL.
3"""
5import contextlib
6import csv
7import datetime as dt
8import inspect
9import json
10import math
11import types
12import typing
13import warnings
14from collections import defaultdict
15from copy import copy
16from decimal import Decimal
17from pathlib import Path
18from typing import Any, Optional
20import pydal
21from pydal._globals import DEFAULT
22from pydal.objects import Field as _Field
23from pydal.objects import Query as _Query
24from pydal.objects import Row
25from pydal.objects import Table as _Table
26from typing_extensions import Self
28from .config import TypeDALConfig, load_config
29from .helpers import (
30 DummyQuery,
31 all_annotations,
32 all_dict,
33 as_lambda,
34 extract_type_optional,
35 filter_out,
36 instanciate,
37 is_union,
38 looks_like,
39 mktable,
40 origin_is_subclass,
41 to_snake,
42 unwrap_type,
43)
44from .serializers import as_json
45from .types import (
46 AfterDeleteCallable,
47 AfterInsertCallable,
48 AfterUpdateCallable,
49 AnyDict,
50 BeforeDeleteCallable,
51 BeforeInsertCallable,
52 BeforeUpdateCallable,
53 CacheMetadata,
54 Expression,
55 Field,
56 Metadata,
57 PaginateDict,
58 Pagination,
59 Query,
60 Rows,
61 Validator,
62 _Types,
63)
65# use typing.cast(type, ...) to make mypy happy with unions
66T_annotation = typing.Type[Any] | types.UnionType
67T_Query = typing.Union["Table", Query, bool, None, "TypedTable", typing.Type["TypedTable"]]
68T_Value = typing.TypeVar("T_Value") # actual type of the Field (via Generic)
69T_MetaInstance = typing.TypeVar("T_MetaInstance", bound="TypedTable") # bound="TypedTable"; bound="TableMeta"
70T = typing.TypeVar("T")
72BASIC_MAPPINGS: dict[T_annotation, str] = {
73 str: "string",
74 int: "integer",
75 bool: "boolean",
76 bytes: "blob",
77 float: "double",
78 object: "json",
79 Decimal: "decimal(10,2)",
80 dt.date: "date",
81 dt.time: "time",
82 dt.datetime: "datetime",
83}
86def is_typed_field(cls: Any) -> typing.TypeGuard["TypedField[Any]"]:
87 """
88 Is `cls` an instance or subclass of TypedField?
90 Deprecated
91 """
92 return (
93 isinstance(cls, TypedField)
94 or isinstance(typing.get_origin(cls), type)
95 and issubclass(typing.get_origin(cls), TypedField)
96 )
99JOIN_OPTIONS = typing.Literal["left", "inner", None]
100DEFAULT_JOIN_OPTION: JOIN_OPTIONS = "left"
102# table-ish paramter:
103P_Table = typing.Union[typing.Type["TypedTable"], pydal.objects.Table]
105Condition: typing.TypeAlias = typing.Optional[
106 typing.Callable[
107 # self, other -> Query
108 [P_Table, P_Table],
109 Query | bool,
110 ]
111]
113OnQuery: typing.TypeAlias = typing.Optional[
114 typing.Callable[
115 # self, other -> list of .on statements
116 [P_Table, P_Table],
117 list[Expression],
118 ]
119]
121To_Type = typing.TypeVar("To_Type", type[Any], typing.Type[Any], str)
124class Relationship(typing.Generic[To_Type]):
125 """
126 Define a relationship to another table.
127 """
129 _type: To_Type
130 table: typing.Type["TypedTable"] | type | str
131 condition: Condition
132 on: OnQuery
133 multiple: bool
134 join: JOIN_OPTIONS
136 def __init__(
137 self,
138 _type: To_Type,
139 condition: Condition = None,
140 join: JOIN_OPTIONS = None,
141 on: OnQuery = None,
142 ):
143 """
144 Should not be called directly, use relationship() instead!
145 """
146 if condition and on:
147 warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}")
148 raise ValueError("Please specify either a condition or an 'on' statement for this relationship!")
150 self._type = _type
151 self.condition = condition
152 self.join = "left" if on else join # .on is always left join!
153 self.on = on
155 if args := typing.get_args(_type):
156 self.table = unwrap_type(args[0])
157 self.multiple = True
158 else:
159 self.table = _type
160 self.multiple = False
162 if isinstance(self.table, str):
163 self.table = TypeDAL.to_snake(self.table)
165 def clone(self, **update: Any) -> "Relationship[To_Type]":
166 """
167 Create a copy of the relationship, possibly updated.
168 """
169 return self.__class__(
170 update.get("_type") or self._type,
171 update.get("condition") or self.condition,
172 update.get("join") or self.join,
173 update.get("on") or self.on,
174 )
176 def __repr__(self) -> str:
177 """
178 Representation of the relationship.
179 """
180 if callback := self.condition or self.on:
181 src_code = inspect.getsource(callback).strip()
182 else:
183 cls_name = self._type if isinstance(self._type, str) else self._type.__name__ # type: ignore
184 src_code = f"to {cls_name} (missing condition)"
186 join = f":{self.join}" if self.join else ""
187 return f"<Relationship{join} {src_code}>"
189 def get_table(self, db: "TypeDAL") -> typing.Type["TypedTable"]:
190 """
191 Get the table this relationship is bound to.
192 """
193 table = self.table # can be a string because db wasn't available yet
194 if isinstance(table, str):
195 if mapped := db._class_map.get(table):
196 # yay
197 return mapped
199 # boo, fall back to untyped table but pretend it is typed:
200 return typing.cast(typing.Type["TypedTable"], db[table]) # eh close enough!
202 return table
204 def get_table_name(self) -> str:
205 """
206 Get the name of the table this relationship is bound to.
207 """
208 if isinstance(self.table, str):
209 return self.table
211 if isinstance(self.table, pydal.objects.Table):
212 return str(self.table)
214 # else: typed table
215 try:
216 table = self.table._ensure_table_defined() if issubclass(self.table, TypedTable) else self.table
217 except Exception: # pragma: no cover
218 table = self.table
220 return str(table)
222 def __get__(self, instance: Any, owner: Any) -> typing.Optional[list[Any]] | "Relationship[To_Type]":
223 """
224 Relationship is a descriptor class, which can be returned from a class but not an instance.
226 For an instance, using .join() will replace the Relationship with the actual data.
227 If you forgot to join, a warning will be shown and empty data will be returned.
228 """
229 if not instance:
230 # relationship queried on class, that's allowed
231 return self
233 warnings.warn(
234 "Trying to get data from a relationship object! Did you forget to join it?", category=RuntimeWarning
235 )
236 if self.multiple:
237 return []
238 else:
239 return None
242def relationship(
243 _type: To_Type, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None
244) -> Relationship[To_Type]:
245 """
246 Define a relationship to another table, when its id is not stored in the current table.
248 Example:
249 class User(TypedTable):
250 name: str
252 posts = relationship(list["Post"], condition=lambda self, post: self.id == post.author, join='left')
254 class Post(TypedTable):
255 title: str
256 author: User
258 User.join("posts").first() # User instance with list[Post] in .posts
260 Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts.
261 In this case, the join strategy is set to LEFT so users without posts are also still selected.
263 For complex queries with a pivot table, a `on` can be set insteaad of `condition`:
264 class User(TypedTable):
265 ...
267 tags = relationship(list["Tag"], on=lambda self, tag: [
268 Tagged.on(Tagged.entity == entity.gid),
269 Tag.on((Tagged.tag == tag.id)),
270 ])
272 If you'd try to capture this in a single 'condition', pydal would create a cross join which is much less efficient.
273 """
274 return Relationship(_type, condition, join, on)
277def _generate_relationship_condition(
278 _: typing.Type["TypedTable"], key: str, field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]]
279) -> Condition:
280 origin = typing.get_origin(field)
281 # else: generic
283 if origin == list:
284 # field = typing.get_args(field)[0] # actual field
285 # return lambda _self, _other: cls[key].contains(field)
287 return lambda _self, _other: _self[key].contains(_other.id)
288 else:
289 # normal reference
290 # return lambda _self, _other: cls[key] == field.id
291 return lambda _self, _other: _self[key] == _other.id
294def to_relationship(
295 cls: typing.Type["TypedTable"] | type[Any],
296 key: str,
297 field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]],
298) -> typing.Optional[Relationship[Any]]:
299 """
300 Used to automatically create relationship instance for reference fields.
302 Example:
303 class MyTable(TypedTable):
304 reference: OtherTable
306 `reference` contains the id of an Other Table row.
307 MyTable.relationships should have 'reference' as a relationship, so `MyTable.join('reference')` should work.
309 This function will automatically perform this logic (called in db.define):
310 to_relationship(MyTable, 'reference', OtherTable) -> Relationship[OtherTable]
312 Also works for list:reference (list[OtherTable]) and TypedField[OtherTable].
313 """
314 if looks_like(field, TypedField):
315 if args := typing.get_args(field):
316 field = args[0]
317 else:
318 # weird
319 return None
321 field, optional = extract_type_optional(field)
323 try:
324 condition = _generate_relationship_condition(cls, key, field)
325 except Exception as e: # pragma: no cover
326 warnings.warn("Could not generate Relationship condition", source=e)
327 condition = None
329 if not condition: # pragma: no cover
330 # something went wrong, not a valid relationship
331 warnings.warn(f"Invalid relationship for {cls.__name__}.{key}: {field}")
332 return None
334 join = "left" if optional or typing.get_origin(field) == list else "inner"
336 return Relationship(typing.cast(type[TypedTable], field), condition, typing.cast(JOIN_OPTIONS, join))
339class TypeDAL(pydal.DAL): # type: ignore
340 """
341 Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables.
342 """
344 _config: TypeDALConfig
346 def __init__(
347 self,
348 uri: Optional[str] = None, # default from config or 'sqlite:memory'
349 pool_size: int = None, # default 1 if sqlite else 3
350 folder: Optional[str | Path] = None, # default 'databases' in config
351 db_codec: str = "UTF-8",
352 check_reserved: Optional[list[str]] = None,
353 migrate: Optional[bool] = None, # default True by config
354 fake_migrate: Optional[bool] = None, # default False by config
355 migrate_enabled: bool = True,
356 fake_migrate_all: bool = False,
357 decode_credentials: bool = False,
358 driver_args: Optional[AnyDict] = None,
359 adapter_args: Optional[AnyDict] = None,
360 attempts: int = 5,
361 auto_import: bool = False,
362 bigint_id: bool = False,
363 debug: bool = False,
364 lazy_tables: bool = False,
365 db_uid: Optional[str] = None,
366 after_connection: typing.Callable[..., Any] = None,
367 tables: Optional[list[str]] = None,
368 ignore_field_case: bool = True,
369 entity_quoting: bool = True,
370 table_hash: Optional[str] = None,
371 enable_typedal_caching: bool = None,
372 use_pyproject: bool | str = True,
373 use_env: bool | str = True,
374 connection: Optional[str] = None,
375 config: Optional[TypeDALConfig] = None,
376 ) -> None:
377 """
378 Adds some internal tables after calling pydal's default init.
380 Set enable_typedal_caching to False to disable this behavior.
381 """
382 config = config or load_config(connection, _use_pyproject=use_pyproject, _use_env=use_env)
383 config.update(
384 database=uri,
385 dialect=uri.split(":")[0] if uri and ":" in uri else None,
386 folder=str(folder) if folder is not None else None,
387 migrate=migrate,
388 fake_migrate=fake_migrate,
389 caching=enable_typedal_caching,
390 pool_size=pool_size,
391 )
393 self._config = config
395 if config.folder:
396 Path(config.folder).mkdir(exist_ok=True)
398 super().__init__(
399 config.database,
400 config.pool_size,
401 config.folder,
402 db_codec,
403 check_reserved,
404 config.migrate,
405 config.fake_migrate,
406 migrate_enabled,
407 fake_migrate_all,
408 decode_credentials,
409 driver_args,
410 adapter_args,
411 attempts,
412 auto_import,
413 bigint_id,
414 debug,
415 lazy_tables,
416 db_uid,
417 after_connection,
418 tables,
419 ignore_field_case,
420 entity_quoting,
421 table_hash,
422 )
424 if config.caching:
425 self.try_define(_TypedalCache)
426 self.try_define(_TypedalCacheDependency)
428 def try_define(self, model: typing.Type[T], verbose: bool = False) -> typing.Type[T]:
429 """
430 Try to define a model with migrate or fall back to fake migrate.
431 """
432 try:
433 return self.define(model, migrate=True)
434 except Exception as e:
435 # clean up:
436 self.rollback()
437 if (tablename := self.to_snake(model.__name__)) and tablename in dir(self):
438 delattr(self, tablename)
440 if verbose:
441 warnings.warn(f"{model} could not be migrated, try faking", source=e, category=RuntimeWarning)
443 # try again:
444 return self.define(model, migrate=True, fake_migrate=True, redefine=True)
446 default_kwargs: typing.ClassVar[AnyDict] = {
447 # fields are 'required' (notnull) by default:
448 "notnull": True,
449 }
451 # maps table name to typedal class, for resolving future references
452 _class_map: typing.ClassVar[dict[str, typing.Type["TypedTable"]]] = {}
454 def _define(self, cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:
455 # todo: new relationship item added should also invalidate (previously unrelated) cache result
457 # todo: option to enable/disable cache dependency behavior:
458 # - don't set _before_update and _before_delete
459 # - don't add TypedalCacheDependency entry
460 # - don't invalidate other item on new row of this type
462 # when __future__.annotations is implemented, cls.__annotations__ will not work anymore as below.
463 # proper way to handle this would be (but gives error right now due to Table implementing magic methods):
464 # typing.get_type_hints(cls, globalns=None, localns=None)
466 # dirty way (with evil eval):
467 # [eval(v) for k, v in cls.__annotations__.items()]
468 # this however also stops working when variables outside this scope or even references to other
469 # objects are used. So for now, this package will NOT work when from __future__ import annotations is used,
470 # and might break in the future, when this annotations behavior is enabled by default.
472 # non-annotated variables have to be passed to define_table as kwargs
473 full_dict = all_dict(cls) # includes properties from parents (e.g. useful for mixins)
475 tablename = self.to_snake(cls.__name__)
476 # grab annotations of cls and it's parents:
477 annotations = all_annotations(cls)
478 # extend with `prop = TypedField()` 'annotations':
479 annotations |= {k: typing.cast(type, v) for k, v in full_dict.items() if is_typed_field(v)}
480 # remove internal stuff:
481 annotations = {k: v for k, v in annotations.items() if not k.startswith("_")}
483 typedfields: dict[str, TypedField[Any]] = {
484 k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v)
485 }
487 relationships: dict[str, type[Relationship[Any]]] = filter_out(annotations, Relationship)
489 fields = {fname: self._to_field(fname, ftype) for fname, ftype in annotations.items()}
491 # ! dont' use full_dict here:
492 other_kwargs = kwargs | {
493 k: v for k, v in cls.__dict__.items() if k not in annotations and not k.startswith("_")
494 } # other_kwargs was previously used to pass kwargs to typedal, but use @define(**kwargs) for that.
495 # now it's only used to extract relationships from the object.
496 # other properties of the class (incl methods) should not be touched
498 # for key in typedfields.keys() - full_dict.keys():
499 # # typed fields that don't haven't been added to the object yet
500 # setattr(cls, key, typedfields[key])
502 for key, field in typedfields.items():
503 # clone every property so it can be re-used across mixins:
504 clone = copy(field)
505 setattr(cls, key, clone)
506 typedfields[key] = clone
508 # start with base classes and overwrite with current class:
509 relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship)
511 # DEPRECATED: Relationship as annotation is currently not supported!
512 # ensure they are all instances and
513 # not mix of instances (`= relationship()`) and classes (`: Relationship[...]`):
514 # relationships = {
515 # k: v if isinstance(v, Relationship) else to_relationship(cls, k, v) for k, v in relationships.items()
516 # }
518 # keys of implicit references (also relationships):
519 reference_field_keys = [k for k, v in fields.items() if v.type.split(" ")[0] in ("list:reference", "reference")]
521 # add implicit relationships:
522 # User; list[User]; TypedField[User]; TypedField[list[User]]
523 relationships |= {
524 k: new_relationship
525 for k in reference_field_keys
526 if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k]))
527 }
529 cache_dependency = self._config.caching and kwargs.pop("cache_dependency", True)
531 table: Table = self.define_table(tablename, *fields.values(), **kwargs)
533 for name, typed_field in typedfields.items():
534 field = fields[name]
535 typed_field.bind(field, table)
537 if issubclass(cls, TypedTable):
538 cls.__set_internals__(
539 db=self,
540 table=table,
541 # by now, all relationships should be instances!
542 relationships=typing.cast(dict[str, Relationship[Any]], relationships),
543 )
544 self._class_map[str(table)] = cls
545 cls.__on_define__(self)
546 else:
547 warnings.warn("db.define used without inheriting TypedTable. This could lead to strange problems!")
549 if not tablename.startswith("typedal_") and cache_dependency:
550 table._before_update.append(lambda s, _: _remove_cache(s, tablename))
551 table._before_delete.append(lambda s: _remove_cache(s, tablename))
553 return cls
555 @typing.overload
556 def define(self, maybe_cls: None = None, **kwargs: Any) -> typing.Callable[[typing.Type[T]], typing.Type[T]]:
557 """
558 Typing Overload for define without a class.
560 @db.define()
561 class MyTable(TypedTable): ...
562 """
564 @typing.overload
565 def define(self, maybe_cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:
566 """
567 Typing Overload for define with a class.
569 @db.define
570 class MyTable(TypedTable): ...
571 """
573 def define(
574 self, maybe_cls: typing.Type[T] | None = None, **kwargs: Any
575 ) -> typing.Type[T] | typing.Callable[[typing.Type[T]], typing.Type[T]]:
576 """
577 Can be used as a decorator on a class that inherits `TypedTable`, \
578 or as a regular method if you need to define your classes before you have access to a 'db' instance.
580 You can also pass extra arguments to db.define_table.
581 See http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Table-constructor
583 Example:
584 @db.define
585 class Person(TypedTable):
586 ...
588 class Article(TypedTable):
589 ...
591 # at a later time:
592 db.define(Article)
594 Returns:
595 the result of pydal.define_table
596 """
598 def wrapper(cls: typing.Type[T]) -> typing.Type[T]:
599 return self._define(cls, **kwargs)
601 if maybe_cls:
602 return wrapper(maybe_cls)
604 return wrapper
606 # def drop(self, table_name: str) -> None:
607 # """
608 # Remove a table by name (both on the database level and the typedal level).
609 # """
610 # # drop calls TypedTable.drop() and removes it from the `_class_map`
611 # if cls := self._class_map.pop(table_name, None):
612 # cls.drop()
614 # def drop_all(self, max_retries: int = None) -> None:
615 # """
616 # Remove all tables and keep doing so until everything is gone!
617 # """
618 # retries = 0
619 # if max_retries is None:
620 # max_retries = len(self.tables)
621 #
622 # while self.tables:
623 # retries += 1
624 # for table in self.tables:
625 # self.drop(table)
626 #
627 # if retries > max_retries:
628 # raise RuntimeError("Could not delete all tables")
630 def __call__(self, *_args: T_Query, **kwargs: Any) -> "TypedSet":
631 """
632 A db instance can be called directly to perform a query.
634 Usually, only a query is passed.
636 Example:
637 db(query).select()
639 """
640 args = list(_args)
641 if args:
642 cls = args[0]
643 if isinstance(cls, bool):
644 raise ValueError("Don't actually pass a bool to db()! Use a query instead.")
646 if isinstance(cls, type) and issubclass(type(cls), type) and issubclass(cls, TypedTable):
647 # table defined without @db.define decorator!
648 _cls: typing.Type[TypedTable] = cls
649 args[0] = _cls.id != None
651 _set = super().__call__(*args, **kwargs)
652 return typing.cast(TypedSet, _set)
654 def __getitem__(self, key: str) -> "Table":
655 """
656 Allows dynamically accessing a table by its name as a string.
658 Example:
659 db['users'] -> user
660 """
661 return typing.cast(Table, super().__getitem__(str(key)))
663 @classmethod
664 def _build_field(cls, name: str, _type: str, **kw: Any) -> Field:
665 return Field(name, _type, **{**cls.default_kwargs, **kw})
667 @classmethod
668 def _annotation_to_pydal_fieldtype(
669 cls, _ftype: T_annotation, mut_kw: typing.MutableMapping[str, Any]
670 ) -> Optional[str]:
671 # ftype can be a union or type. typing.cast is sometimes used to tell mypy when it's not a union.
672 ftype = typing.cast(type, _ftype) # cast from typing.Type to type to make mypy happy)
674 if isinstance(ftype, str):
675 # extract type from string
676 ftype = typing.get_args(typing.Type[ftype])[0]._evaluate(
677 localns=locals(), globalns=globals(), recursive_guard=frozenset()
678 )
680 if mapping := BASIC_MAPPINGS.get(ftype):
681 # basi types
682 return mapping
683 elif isinstance(ftype, _Table):
684 # db.table
685 return f"reference {ftype._tablename}"
686 elif issubclass(type(ftype), type) and issubclass(ftype, TypedTable):
687 # SomeTable
688 snakename = cls.to_snake(ftype.__name__)
689 return f"reference {snakename}"
690 elif isinstance(ftype, TypedField):
691 # FieldType(type, ...)
692 return ftype._to_field(mut_kw)
693 elif origin_is_subclass(ftype, TypedField):
694 # TypedField[int]
695 return cls._annotation_to_pydal_fieldtype(typing.get_args(ftype)[0], mut_kw)
696 elif isinstance(ftype, types.GenericAlias) and typing.get_origin(ftype) in (list, TypedField):
697 # list[str] -> str -> string -> list:string
698 _child_type = typing.get_args(ftype)[0]
699 _child_type = cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)
700 return f"list:{_child_type}"
701 elif is_union(ftype):
702 # str | int -> UnionType
703 # typing.Union[str | int] -> typing._UnionGenericAlias
705 # Optional[type] == type | None
707 match typing.get_args(ftype):
708 case (_child_type, _Types.NONETYPE) | (_Types.NONETYPE, _child_type):
709 # good union of Nullable
711 # if a field is optional, it is nullable:
712 mut_kw["notnull"] = False
713 return cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)
714 case _:
715 # two types is not supported by the db!
716 return None
717 else:
718 return None
720 @classmethod
721 def _to_field(cls, fname: str, ftype: type, **kw: Any) -> Field:
722 """
723 Convert a annotation into a pydal Field.
725 Args:
726 fname: name of the property
727 ftype: annotation of the property
728 kw: when using TypedField or a function returning it (e.g. StringField),
729 keyword args can be used to pass any other settings you would normally to a pydal Field
731 -> pydal.Field(fname, ftype, **kw)
733 Example:
734 class MyTable:
735 fname: ftype
736 id: int
737 name: str
738 reference: Table
739 other: TypedField(str, default="John Doe") # default will be in kwargs
740 """
741 fname = cls.to_snake(fname)
743 if converted_type := cls._annotation_to_pydal_fieldtype(ftype, kw):
744 return cls._build_field(fname, converted_type, **kw)
745 else:
746 raise NotImplementedError(f"Unsupported type {ftype}/{type(ftype)}")
748 @staticmethod
749 def to_snake(camel: str) -> str:
750 """
751 Moved to helpers, kept as a static method for legacy reasons.
752 """
753 return to_snake(camel)
756class TableProtocol(typing.Protocol): # pragma: no cover
757 """
758 Make mypy happy.
759 """
761 id: "TypedField[int]"
763 def __getitem__(self, item: str) -> Field:
764 """
765 Tell mypy a Table supports dictionary notation for columns.
766 """
769class Table(_Table, TableProtocol): # type: ignore
770 """
771 Make mypy happy.
772 """
775class TableMeta(type):
776 """
777 This metaclass contains functionality on table classes, that doesn't exist on its instances.
779 Example:
780 class MyTable(TypedTable):
781 some_field: TypedField[int]
783 MyTable.update_or_insert(...) # should work
785 MyTable.some_field # -> Field, can be used to query etc.
787 row = MyTable.first() # returns instance of MyTable
789 # row.update_or_insert(...) # shouldn't work!
791 row.some_field # -> int, with actual data
793 """
795 # set up by db.define:
796 # _db: TypeDAL | None = None
797 # _table: Table | None = None
798 _db: TypeDAL | None = None
799 _table: Table | None = None
800 _relationships: dict[str, Relationship[Any]] | None = None
802 #########################
803 # TypeDAL custom logic: #
804 #########################
806 def __set_internals__(self, db: pydal.DAL, table: Table, relationships: dict[str, Relationship[Any]]) -> None:
807 """
808 Store the related database and pydal table for later usage.
809 """
810 self._db = db
811 self._table = table
812 self._relationships = relationships
814 def __getattr__(self, col: str) -> Optional[Field]:
815 """
816 Magic method used by TypedTableMeta to get a database field with dot notation on a class.
818 Example:
819 SomeTypedTable.col -> db.table.col (via TypedTableMeta.__getattr__)
821 """
822 if self._table:
823 return getattr(self._table, col, None)
825 return None
827 def _ensure_table_defined(self) -> Table:
828 if not self._table:
829 raise EnvironmentError("@define or db.define is not called on this class yet!")
830 return self._table
832 def __iter__(self) -> typing.Generator[Field, None, None]:
833 """
834 Loop through the columns of this model.
835 """
836 table = self._ensure_table_defined()
837 yield from iter(table)
839 def __getitem__(self, item: str) -> Field:
840 """
841 Allow dict notation to get a column of this table (-> Field instance).
842 """
843 table = self._ensure_table_defined()
844 return table[item]
846 def __str__(self) -> str:
847 """
848 Normally, just returns the underlying table name, but with a fallback if the model is unbound.
849 """
850 if self._table:
851 return str(self._table)
852 else:
853 return f"<unbound table {self.__name__}>"
855 def from_row(self: typing.Type[T_MetaInstance], row: pydal.objects.Row) -> T_MetaInstance:
856 """
857 Create a model instance from a pydal row.
858 """
859 return self(row)
861 def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]":
862 """
863 Return all rows for this model.
864 """
865 return self.collect()
867 def get_relationships(self) -> dict[str, Relationship[Any]]:
868 """
869 Return the registered relationships of the current model.
870 """
871 return self._relationships or {}
873 ##########################
874 # TypeDAL Modified Logic #
875 ##########################
877 def insert(self: typing.Type[T_MetaInstance], **fields: Any) -> T_MetaInstance:
878 """
879 This is only called when db.define is not used as a decorator.
881 cls.__table functions as 'self'
883 Args:
884 **fields: anything you want to insert in the database
886 Returns: the ID of the new row.
888 """
889 table = self._ensure_table_defined()
891 result = table.insert(**fields)
892 # it already is an int but mypy doesn't understand that
893 return self(result)
895 def _insert(self, **fields: Any) -> str:
896 table = self._ensure_table_defined()
898 return str(table._insert(**fields))
900 def bulk_insert(self: typing.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRows[T_MetaInstance]":
901 """
902 Insert multiple rows, returns a TypedRows set of new instances.
903 """
904 table = self._ensure_table_defined()
905 result = table.bulk_insert(items)
906 return self.where(lambda row: row.id.belongs(result)).collect()
908 def update_or_insert(
909 self: typing.Type[T_MetaInstance], query: T_Query | AnyDict = DEFAULT, **values: Any
910 ) -> T_MetaInstance:
911 """
912 Update a row if query matches, else insert a new one.
914 Returns the created or updated instance.
915 """
916 table = self._ensure_table_defined()
918 if query is DEFAULT:
919 record = table(**values)
920 elif isinstance(query, dict):
921 record = table(**query)
922 else:
923 record = table(query)
925 if not record:
926 return self.insert(**values)
928 record.update_record(**values)
929 return self(record)
931 def validate_and_insert(
932 self: typing.Type[T_MetaInstance], **fields: Any
933 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:
934 """
935 Validate input data and then insert a row.
937 Returns a tuple of (the created instance, a dict of errors).
938 """
939 table = self._ensure_table_defined()
940 result = table.validate_and_insert(**fields)
941 if row_id := result.get("id"):
942 return self(row_id), None
943 else:
944 return None, result.get("errors")
946 def validate_and_update(
947 self: typing.Type[T_MetaInstance], query: Query, **fields: Any
948 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:
949 """
950 Validate input data and then update max 1 row.
952 Returns a tuple of (the updated instance, a dict of errors).
953 """
954 table = self._ensure_table_defined()
956 result = table.validate_and_update(query, **fields)
958 if errors := result.get("errors"):
959 return None, errors
960 elif row_id := result.get("id"):
961 return self(row_id), None
962 else: # pragma: no cover
963 # update on query without result (shouldnt happen)
964 return None, None
966 def validate_and_update_or_insert(
967 self: typing.Type[T_MetaInstance], query: Query, **fields: Any
968 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:
969 """
970 Validate input data and then update_and_insert (on max 1 row).
972 Returns a tuple of (the updated/created instance, a dict of errors).
973 """
974 table = self._ensure_table_defined()
975 result = table.validate_and_update_or_insert(query, **fields)
977 if errors := result.get("errors"):
978 return None, errors
979 elif row_id := result.get("id"):
980 return self(row_id), None
981 else: # pragma: no cover
982 # update on query without result (shouldnt happen)
983 return None, None
985 def select(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":
986 """
987 See QueryBuilder.select!
988 """
989 return QueryBuilder(self).select(*a, **kw)
991 def paginate(self: typing.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]":
992 """
993 See QueryBuilder.paginate!
994 """
995 return QueryBuilder(self).paginate(limit=limit, page=page)
997 def chunk(
998 self: typing.Type[T_MetaInstance], chunk_size: int
999 ) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:
1000 """
1001 See QueryBuilder.chunk!
1002 """
1003 return QueryBuilder(self).chunk(chunk_size)
1005 def where(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":
1006 """
1007 See QueryBuilder.where!
1008 """
1009 return QueryBuilder(self).where(*a, **kw)
1011 def cache(self: typing.Type[T_MetaInstance], *deps: Any, **kwargs: Any) -> "QueryBuilder[T_MetaInstance]":
1012 """
1013 See QueryBuilder.cache!
1014 """
1015 return QueryBuilder(self).cache(*deps, **kwargs)
1017 def count(self: typing.Type[T_MetaInstance]) -> int:
1018 """
1019 See QueryBuilder.count!
1020 """
1021 return QueryBuilder(self).count()
1023 def first(self: typing.Type[T_MetaInstance]) -> T_MetaInstance | None:
1024 """
1025 See QueryBuilder.first!
1026 """
1027 return QueryBuilder(self).first()
1029 def join(
1030 self: typing.Type[T_MetaInstance],
1031 *fields: str | typing.Type["TypedTable"],
1032 method: JOIN_OPTIONS = None,
1033 on: OnQuery | list[Expression] | Expression = None,
1034 condition: Condition = None,
1035 ) -> "QueryBuilder[T_MetaInstance]":
1036 """
1037 See QueryBuilder.join!
1038 """
1039 return QueryBuilder(self).join(*fields, on=on, condition=condition, method=method)
1041 def collect(self: typing.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]":
1042 """
1043 See QueryBuilder.collect!
1044 """
1045 return QueryBuilder(self).collect(verbose=verbose)
1047 @property
1048 def ALL(cls) -> pydal.objects.SQLALL:
1049 """
1050 Select all fields for this table.
1051 """
1052 table = cls._ensure_table_defined()
1054 return table.ALL
1056 ##########################
1057 # TypeDAL Shadowed Logic #
1058 ##########################
1059 fields: list[str]
1061 # other table methods:
1063 def truncate(self, mode: str = "") -> None:
1064 """
1065 Remove all data and reset index.
1066 """
1067 table = self._ensure_table_defined()
1068 table.truncate(mode)
1070 def drop(self, mode: str = "") -> None:
1071 """
1072 Remove the underlying table.
1073 """
1074 table = self._ensure_table_defined()
1075 table.drop(mode)
1077 def create_index(self, name: str, *fields: Field | str, **kwargs: Any) -> bool:
1078 """
1079 Add an index on some columns of this table.
1080 """
1081 table = self._ensure_table_defined()
1082 result = table.create_index(name, *fields, **kwargs)
1083 return typing.cast(bool, result)
1085 def drop_index(self, name: str, if_exists: bool = False) -> bool:
1086 """
1087 Remove an index from this table.
1088 """
1089 table = self._ensure_table_defined()
1090 result = table.drop_index(name, if_exists)
1091 return typing.cast(bool, result)
1093 def import_from_csv_file(
1094 self,
1095 csvfile: typing.TextIO,
1096 id_map: dict[str, str] = None,
1097 null: Any = "<NULL>",
1098 unique: str = "uuid",
1099 id_offset: dict[str, int] = None, # id_offset used only when id_map is None
1100 transform: typing.Callable[[dict[Any, Any]], dict[Any, Any]] = None,
1101 validate: bool = False,
1102 encoding: str = "utf-8",
1103 delimiter: str = ",",
1104 quotechar: str = '"',
1105 quoting: int = csv.QUOTE_MINIMAL,
1106 restore: bool = False,
1107 **kwargs: Any,
1108 ) -> None:
1109 """
1110 Load a csv file into the database.
1111 """
1112 table = self._ensure_table_defined()
1113 table.import_from_csv_file(
1114 csvfile,
1115 id_map=id_map,
1116 null=null,
1117 unique=unique,
1118 id_offset=id_offset,
1119 transform=transform,
1120 validate=validate,
1121 encoding=encoding,
1122 delimiter=delimiter,
1123 quotechar=quotechar,
1124 quoting=quoting,
1125 restore=restore,
1126 **kwargs,
1127 )
1129 def on(self, query: Query | bool) -> Expression:
1130 """
1131 Shadow Table.on.
1133 Used for joins.
1135 See Also:
1136 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation
1137 """
1138 table = self._ensure_table_defined()
1139 return typing.cast(Expression, table.on(query))
1141 def with_alias(self, alias: str) -> _Table:
1142 """
1143 Shadow Table.with_alias.
1145 Useful for joins when joining the same table multiple times.
1147 See Also:
1148 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation
1149 """
1150 table = self._ensure_table_defined()
1151 return table.with_alias(alias)
1153 # @typing.dataclass_transform()
1156class TypedField(typing.Generic[T_Value]): # pragma: no cover
1157 """
1158 Typed version of pydal.Field, which will be converted to a normal Field in the background.
1159 """
1161 # will be set by .bind on db.define
1162 name = ""
1163 _db: Optional[pydal.DAL] = None
1164 _rname: Optional[str] = None
1165 _table: Optional[Table] = None
1166 _field: Optional[Field] = None
1168 _type: T_annotation
1169 kwargs: Any
1171 requires: Validator | typing.Iterable[Validator]
1173 def __init__(self, _type: typing.Type[T_Value] | types.UnionType = str, /, **settings: Any) -> None: # type: ignore
1174 """
1175 A TypedFieldType should not be inited manually, but TypedField (from `fields.py`) should be used!
1176 """
1177 self._type = _type
1178 self.kwargs = settings
1179 super().__init__()
1181 @typing.overload
1182 def __get__(self, instance: T_MetaInstance, owner: typing.Type[T_MetaInstance]) -> T_Value: # pragma: no cover
1183 """
1184 row.field -> (actual data).
1185 """
1187 @typing.overload
1188 def __get__(self, instance: None, owner: "typing.Type[TypedTable]") -> "TypedField[T_Value]": # pragma: no cover
1189 """
1190 Table.field -> Field.
1191 """
1193 def __get__(
1194 self, instance: T_MetaInstance | None, owner: typing.Type[T_MetaInstance]
1195 ) -> typing.Union[T_Value, "TypedField[T_Value]"]:
1196 """
1197 Since this class is a Descriptor field, \
1198 it returns something else depending on if it's called on a class or instance.
1200 (this is mostly for mypy/typing)
1201 """
1202 if instance:
1203 # this is only reached in a very specific case:
1204 # an instance of the object was created with a specific set of fields selected (excluding the current one)
1205 # in that case, no value was stored in the owner -> return None (since the field was not selected)
1206 return typing.cast(T_Value, None) # cast as T_Value so mypy understands it for selected fields
1207 else:
1208 # getting as class -> return actual field so pydal understands it when using in query etc.
1209 return typing.cast(TypedField[T_Value], self._field) # pretend it's still typed for IDE support
1211 def __str__(self) -> str:
1212 """
1213 String representation of a Typed Field.
1215 If `type` is set explicitly (e.g. TypedField(str, type="text")), that type is used: `TypedField.text`,
1216 otherwise the type annotation is used (e.g. TypedField(str) -> TypedField.str)
1217 """
1218 return str(self._field) if self._field else ""
1220 def __repr__(self) -> str:
1221 """
1222 More detailed string representation of a Typed Field.
1224 Uses __str__ and adds the provided extra options (kwargs) in the representation.
1225 """
1226 s = self.__str__()
1228 if "type" in self.kwargs:
1229 # manual type in kwargs supplied
1230 t = self.kwargs["type"]
1231 elif issubclass(type, type(self._type)):
1232 # normal type, str.__name__ = 'str'
1233 t = getattr(self._type, "__name__", str(self._type))
1234 elif t_args := typing.get_args(self._type):
1235 # list[str] -> 'str'
1236 t = t_args[0].__name__
1237 else: # pragma: no cover
1238 # fallback - something else, may not even happen, I'm not sure
1239 t = self._type
1241 s = f"TypedField[{t}].{s}" if s else f"TypedField[{t}]"
1243 kw = self.kwargs.copy()
1244 kw.pop("type", None)
1245 return f"<{s} with options {kw}>"
1247 def _to_field(self, extra_kwargs: typing.MutableMapping[str, Any]) -> Optional[str]:
1248 """
1249 Convert a Typed Field instance to a pydal.Field.
1250 """
1251 other_kwargs = self.kwargs.copy()
1252 extra_kwargs.update(other_kwargs)
1253 return extra_kwargs.pop("type", False) or TypeDAL._annotation_to_pydal_fieldtype(self._type, extra_kwargs)
1255 def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None:
1256 """
1257 Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`.
1258 """
1259 self._table = table
1260 self._field = field
1262 def __getattr__(self, key: str) -> Any:
1263 """
1264 If the regular getattribute does not work, try to get info from the related Field.
1265 """
1266 with contextlib.suppress(AttributeError):
1267 return super().__getattribute__(key)
1269 # try on actual field:
1270 return getattr(self._field, key)
1272 def __eq__(self, other: Any) -> Query:
1273 """
1274 Performing == on a Field will result in a Query.
1275 """
1276 return typing.cast(Query, self._field == other)
1278 def __ne__(self, other: Any) -> Query:
1279 """
1280 Performing != on a Field will result in a Query.
1281 """
1282 return typing.cast(Query, self._field != other)
1284 def __gt__(self, other: Any) -> Query:
1285 """
1286 Performing > on a Field will result in a Query.
1287 """
1288 return typing.cast(Query, self._field > other)
1290 def __lt__(self, other: Any) -> Query:
1291 """
1292 Performing < on a Field will result in a Query.
1293 """
1294 return typing.cast(Query, self._field < other)
1296 def __ge__(self, other: Any) -> Query:
1297 """
1298 Performing >= on a Field will result in a Query.
1299 """
1300 return typing.cast(Query, self._field >= other)
1302 def __le__(self, other: Any) -> Query:
1303 """
1304 Performing <= on a Field will result in a Query.
1305 """
1306 return typing.cast(Query, self._field <= other)
1308 def __hash__(self) -> int:
1309 """
1310 Shadow Field.__hash__.
1311 """
1312 return hash(self._field)
1314 def __invert__(self) -> Expression:
1315 """
1316 Performing ~ on a Field will result in an Expression.
1317 """
1318 if not self._field: # pragma: no cover
1319 raise ValueError("Unbound Field can not be inverted!")
1321 return typing.cast(Expression, ~self._field)
1324class _TypedTable:
1325 """
1326 This class is a final shared parent between TypedTable and Mixins.
1328 This needs to exist because otherwise the __on_define__ of Mixins are not executed.
1329 Notably, this class exists at a level ABOVE the `metaclass=TableMeta`,
1330 because otherwise typing gets confused when Mixins are used and multiple types could satisfy
1331 generic 'T subclass of TypedTable'
1332 -> Setting 'TypedTable' as the parent for Mixin does not work at runtime (and works semi at type check time)
1333 """
1335 id: "TypedField[int]"
1337 _before_insert: list[BeforeInsertCallable]
1338 _after_insert: list[AfterInsertCallable]
1339 _before_update: list[BeforeUpdateCallable]
1340 _after_update: list[AfterUpdateCallable]
1341 _before_delete: list[BeforeDeleteCallable]
1342 _after_delete: list[AfterDeleteCallable]
1344 @classmethod
1345 def __on_define__(cls, db: TypeDAL) -> None:
1346 """
1347 Method that can be implemented by tables to do an action after db.define is completed.
1349 This can be useful if you need to add something like requires=IS_NOT_IN_DB(db, "table.field"),
1350 where you need a reference to the current database, which may not exist yet when defining the model.
1351 """
1354class TypedTable(_TypedTable, metaclass=TableMeta):
1355 """
1356 Enhanded modeling system on top of pydal's Table that adds typing and additional functionality.
1357 """
1359 # set up by 'new':
1360 _row: Row | None = None
1362 _with: list[str]
1364 def _setup_instance_methods(self) -> None:
1365 self.as_dict = self._as_dict # type: ignore
1366 self.__json__ = self.as_json = self._as_json # type: ignore
1367 # self.as_yaml = self._as_yaml # type: ignore
1368 self.as_xml = self._as_xml # type: ignore
1370 self.update = self._update # type: ignore
1372 self.delete_record = self._delete_record # type: ignore
1373 self.update_record = self._update_record # type: ignore
1375 def __new__(
1376 cls, row_or_id: typing.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, **filters: Any
1377 ) -> "TypedTable":
1378 """
1379 Create a Typed Rows model instance from an existing row, ID or query.
1381 Examples:
1382 MyTable(1)
1383 MyTable(id=1)
1384 MyTable(MyTable.id == 1)
1385 """
1386 table = cls._ensure_table_defined()
1387 inst = super().__new__(cls)
1389 if isinstance(row_or_id, TypedTable):
1390 # existing typed table instance!
1391 return row_or_id
1392 elif isinstance(row_or_id, pydal.objects.Row):
1393 row = row_or_id
1394 elif row_or_id is not None:
1395 row = table(row_or_id, **filters)
1396 elif filters:
1397 row = table(**filters)
1398 else:
1399 # dummy object
1400 return inst
1402 if not row:
1403 return None # type: ignore
1405 inst._row = row
1406 inst.__dict__.update(row)
1407 inst._setup_instance_methods()
1408 return inst
1410 def __iter__(self) -> typing.Generator[Any, None, None]:
1411 """
1412 Allows looping through the columns.
1413 """
1414 row = self._ensure_matching_row()
1415 yield from iter(row)
1417 def __getitem__(self, item: str) -> Any:
1418 """
1419 Allows dictionary notation to get columns.
1420 """
1421 if item in self.__dict__:
1422 return self.__dict__.get(item)
1424 # fallback to lookup in row
1425 if self._row:
1426 return self._row[item]
1428 # nothing found!
1429 raise KeyError(item)
1431 def __getattr__(self, item: str) -> Any:
1432 """
1433 Allows dot notation to get columns.
1434 """
1435 if value := self.get(item):
1436 return value
1438 raise AttributeError(item)
1440 def get(self, item: str, default: Any = None) -> Any:
1441 """
1442 Try to get a column from this instance, else return default.
1443 """
1444 try:
1445 return self.__getitem__(item)
1446 except KeyError:
1447 return default
1449 def __setitem__(self, key: str, value: Any) -> None:
1450 """
1451 Data can both be updated via dot and dict notation.
1452 """
1453 return setattr(self, key, value)
1455 def __int__(self) -> int:
1456 """
1457 Calling int on a model instance will return its id.
1458 """
1459 return getattr(self, "id", 0)
1461 def __bool__(self) -> bool:
1462 """
1463 If the instance has an underlying row with data, it is truthy.
1464 """
1465 return bool(getattr(self, "_row", False))
1467 def _ensure_matching_row(self) -> Row:
1468 if not getattr(self, "_row", None):
1469 raise EnvironmentError("Trying to access non-existant row. Maybe it was deleted or not yet initialized?")
1470 return self._row
1472 def __repr__(self) -> str:
1473 """
1474 String representation of the model instance.
1475 """
1476 model_name = self.__class__.__name__
1477 model_data = {}
1479 if self._row:
1480 model_data = self._row.as_json()
1482 details = model_name
1483 details += f"({model_data})"
1485 if relationships := getattr(self, "_with", []):
1486 details += f" + {relationships}"
1488 return f"<{details}>"
1490 # serialization
1491 # underscore variants work for class instances (set up by _setup_instance_methods)
1493 @classmethod
1494 def as_dict(cls, flat: bool = False, sanitize: bool = True) -> AnyDict:
1495 """
1496 Dump the object to a plain dict.
1498 Can be used as both a class or instance method:
1499 - dumps the table info if it's a class
1500 - dumps the row info if it's an instance (see _as_dict)
1501 """
1502 table = cls._ensure_table_defined()
1503 result = table.as_dict(flat, sanitize)
1504 return typing.cast(AnyDict, result)
1506 @classmethod
1507 def as_json(cls, sanitize: bool = True, indent: Optional[int] = None, **kwargs: Any) -> str:
1508 """
1509 Dump the object to json.
1511 Can be used as both a class or instance method:
1512 - dumps the table info if it's a class
1513 - dumps the row info if it's an instance (see _as_json)
1514 """
1515 data = cls.as_dict(sanitize=sanitize)
1516 return as_json.encode(data, indent=indent, **kwargs)
1518 @classmethod
1519 def as_xml(cls, sanitize: bool = True) -> str: # pragma: no cover
1520 """
1521 Dump the object to xml.
1523 Can be used as both a class or instance method:
1524 - dumps the table info if it's a class
1525 - dumps the row info if it's an instance (see _as_xml)
1526 """
1527 table = cls._ensure_table_defined()
1528 return typing.cast(str, table.as_xml(sanitize))
1530 @classmethod
1531 def as_yaml(cls, sanitize: bool = True) -> str:
1532 """
1533 Dump the object to yaml.
1535 Can be used as both a class or instance method:
1536 - dumps the table info if it's a class
1537 - dumps the row info if it's an instance (see _as_yaml)
1538 """
1539 table = cls._ensure_table_defined()
1540 return typing.cast(str, table.as_yaml(sanitize))
1542 def _as_dict(
1543 self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None
1544 ) -> AnyDict:
1545 row = self._ensure_matching_row()
1547 result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)
1549 def asdict_method(obj: Any) -> Any: # pragma: no cover
1550 if hasattr(obj, "_as_dict"): # typedal
1551 return obj._as_dict()
1552 elif hasattr(obj, "as_dict"): # pydal
1553 return obj.as_dict()
1554 else: # something else??
1555 return obj.__dict__
1557 if _with := getattr(self, "_with", None):
1558 for relationship in _with:
1559 data = self.get(relationship)
1561 if isinstance(data, list):
1562 data = [asdict_method(_) for _ in data]
1563 elif data:
1564 data = asdict_method(data)
1566 result[relationship] = data
1568 return typing.cast(AnyDict, result)
1570 def _as_json(
1571 self,
1572 default: typing.Callable[[Any], Any] = None,
1573 indent: Optional[int] = None,
1574 **kwargs: Any,
1575 ) -> str:
1576 data = self._as_dict()
1577 return as_json.encode(data, default=default, indent=indent, **kwargs)
1579 def _as_xml(self, sanitize: bool = True) -> str: # pragma: no cover
1580 row = self._ensure_matching_row()
1581 return typing.cast(str, row.as_xml(sanitize))
1583 # def _as_yaml(self, sanitize: bool = True) -> str:
1584 # row = self._ensure_matching_row()
1585 # return typing.cast(str, row.as_yaml(sanitize))
1587 def __setattr__(self, key: str, value: Any) -> None:
1588 """
1589 When setting a property on a Typed Table model instance, also update the underlying row.
1590 """
1591 if self._row and key in self._row.__dict__ and not callable(value):
1592 # enables `row.key = value; row.update_record()`
1593 self._row[key] = value
1595 super().__setattr__(key, value)
1597 @classmethod
1598 def update(cls: typing.Type[T_MetaInstance], query: Query, **fields: Any) -> T_MetaInstance | None:
1599 """
1600 Update one record.
1602 Example:
1603 MyTable.update(MyTable.id == 1, name="NewName") -> MyTable
1604 """
1605 # todo: update multiple?
1606 if record := cls(query):
1607 return record.update_record(**fields)
1608 else:
1609 return None
1611 def _update(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:
1612 row = self._ensure_matching_row()
1613 row.update(**fields)
1614 self.__dict__.update(**fields)
1615 return self
1617 def _update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:
1618 row = self._ensure_matching_row()
1619 new_row = row.update_record(**fields)
1620 self.update(**new_row)
1621 return self
1623 def update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance: # pragma: no cover
1624 """
1625 Here as a placeholder for _update_record.
1627 Will be replaced on instance creation!
1628 """
1629 return self._update_record(**fields)
1631 def _delete_record(self) -> int:
1632 """
1633 Actual logic in `pydal.helpers.classes.RecordDeleter`.
1634 """
1635 row = self._ensure_matching_row()
1636 result = row.delete_record()
1637 self.__dict__ = {} # empty self, since row is no more.
1638 self._row = None # just to be sure
1639 self._setup_instance_methods()
1640 # ^ instance methods might've been deleted by emptying dict,
1641 # but we still want .as_dict to show an error, not the table's as_dict.
1642 return typing.cast(int, result)
1644 def delete_record(self) -> int: # pragma: no cover
1645 """
1646 Here as a placeholder for _delete_record.
1648 Will be replaced on instance creation!
1649 """
1650 return self._delete_record()
1652 # __del__ is also called on the end of a scope so don't remove records on every del!!
1654 # pickling:
1656 def __getstate__(self) -> AnyDict:
1657 """
1658 State to save when pickling.
1660 Prevents db connection from being pickled.
1661 Similar to as_dict but without changing the data of the relationships (dill does that recursively)
1662 """
1663 row = self._ensure_matching_row()
1664 result: AnyDict = row.as_dict()
1666 if _with := getattr(self, "_with", None):
1667 result["_with"] = _with
1668 for relationship in _with:
1669 data = self.get(relationship)
1671 result[relationship] = data
1673 result["_row"] = self._row.as_json() if self._row else ""
1674 return result
1676 def __setstate__(self, state: AnyDict) -> None:
1677 """
1678 Used by dill when loading from a bytestring.
1679 """
1680 # as_dict also includes table info, so dump as json to only get the actual row data
1681 # then create a new (more empty) row object:
1682 state["_row"] = Row(json.loads(state["_row"]))
1683 self.__dict__ |= state
1686# backwards compat:
1687TypedRow = TypedTable
1690class TypedRows(typing.Collection[T_MetaInstance], Rows):
1691 """
1692 Slighly enhaned and typed functionality on top of pydal Rows (the result of a select).
1693 """
1695 records: dict[int, T_MetaInstance]
1696 # _rows: Rows
1697 model: typing.Type[T_MetaInstance]
1698 metadata: Metadata
1700 # pseudo-properties: actually stored in _rows
1701 db: TypeDAL
1702 colnames: list[str]
1703 fields: list[Field]
1704 colnames_fields: list[Field]
1705 response: list[tuple[Any, ...]]
1707 def __init__(
1708 self,
1709 rows: Rows,
1710 model: typing.Type[T_MetaInstance],
1711 records: dict[int, T_MetaInstance] = None,
1712 metadata: Metadata = None,
1713 ) -> None:
1714 """
1715 Should not be called manually!
1717 Normally, the `records` from an existing `Rows` object are used
1718 but these can be overwritten with a `records` dict.
1719 `metadata` can be any (un)structured data
1720 `model` is a Typed Table class
1721 """
1722 records = records or {row.id: model(row) for row in rows}
1723 super().__init__(rows.db, records, rows.colnames, rows.compact, rows.response, rows.fields)
1724 self.model = model
1725 self.metadata = metadata or {}
1726 self.colnames = rows.colnames
1728 def __len__(self) -> int:
1729 """
1730 Return the count of rows.
1731 """
1732 return len(self.records)
1734 def __iter__(self) -> typing.Iterator[T_MetaInstance]:
1735 """
1736 Loop through the rows.
1737 """
1738 yield from self.records.values()
1740 def __contains__(self, ind: Any) -> bool:
1741 """
1742 Check if an id exists in this result set.
1743 """
1744 return ind in self.records
1746 def first(self) -> T_MetaInstance | None:
1747 """
1748 Get the row with the lowest id.
1749 """
1750 if not self.records:
1751 return None
1753 return next(iter(self))
1755 def last(self) -> T_MetaInstance | None:
1756 """
1757 Get the row with the highest id.
1758 """
1759 if not self.records:
1760 return None
1762 max_id = max(self.records.keys())
1763 return self[max_id]
1765 def find(
1766 self, f: typing.Callable[[T_MetaInstance], Query], limitby: tuple[int, int] = None
1767 ) -> "TypedRows[T_MetaInstance]":
1768 """
1769 Returns a new Rows object, a subset of the original object, filtered by the function `f`.
1770 """
1771 if not self.records:
1772 return self.__class__(self, self.model, {})
1774 records = {}
1775 if limitby:
1776 _min, _max = limitby
1777 else:
1778 _min, _max = 0, len(self)
1779 count = 0
1780 for i, row in self.records.items():
1781 if f(row):
1782 if _min <= count:
1783 records[i] = row
1784 count += 1
1785 if count == _max:
1786 break
1788 return self.__class__(self, self.model, records)
1790 def exclude(self, f: typing.Callable[[T_MetaInstance], Query]) -> "TypedRows[T_MetaInstance]":
1791 """
1792 Removes elements from the calling Rows object, filtered by the function `f`, \
1793 and returns a new Rows object containing the removed elements.
1794 """
1795 if not self.records:
1796 return self.__class__(self, self.model, {})
1797 removed = {}
1798 to_remove = []
1799 for i in self.records:
1800 row = self[i]
1801 if f(row):
1802 removed[i] = self.records[i]
1803 to_remove.append(i)
1805 [self.records.pop(i) for i in to_remove]
1807 return self.__class__(
1808 self,
1809 self.model,
1810 removed,
1811 )
1813 def sort(self, f: typing.Callable[[T_MetaInstance], Any], reverse: bool = False) -> list[T_MetaInstance]:
1814 """
1815 Returns a list of sorted elements (not sorted in place).
1816 """
1817 return [r for (r, s) in sorted(zip(self.records.values(), self), key=lambda r: f(r[1]), reverse=reverse)]
1819 def __str__(self) -> str:
1820 """
1821 Simple string representation.
1822 """
1823 return f"<TypedRows with {len(self)} records>"
1825 def __repr__(self) -> str:
1826 """
1827 Print a table on repr().
1828 """
1829 data = self.as_dict()
1830 headers = list(next(iter(data.values())).keys())
1831 return mktable(data, headers)
1833 def group_by_value(
1834 self, *fields: "str | Field | TypedField[T]", one_result: bool = False, **kwargs: Any
1835 ) -> dict[T, list[T_MetaInstance]]:
1836 """
1837 Group the rows by a specific field (which will be the dict key).
1838 """
1839 kwargs["one_result"] = one_result
1840 result = super().group_by_value(*fields, **kwargs)
1841 return typing.cast(dict[T, list[T_MetaInstance]], result)
1843 def column(self, column: str = None) -> list[Any]:
1844 """
1845 Get a list of all values in a specific column.
1847 Example:
1848 rows.column('name') -> ['Name 1', 'Name 2', ...]
1849 """
1850 return typing.cast(list[Any], super().column(column))
1852 def as_csv(self) -> str:
1853 """
1854 Dump the data to csv.
1855 """
1856 return typing.cast(str, super().as_csv())
1858 def as_dict(
1859 self,
1860 key: str = None,
1861 compact: bool = False,
1862 storage_to_dict: bool = False,
1863 datetime_to_str: bool = False,
1864 custom_types: list[type] = None,
1865 ) -> dict[int, AnyDict]:
1866 """
1867 Get the data in a dict of dicts.
1868 """
1869 if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):
1870 # functionality not guaranteed
1871 return typing.cast(
1872 dict[int, AnyDict],
1873 super().as_dict(
1874 key or "id",
1875 compact,
1876 storage_to_dict,
1877 datetime_to_str,
1878 custom_types,
1879 ),
1880 )
1882 return {k: v.as_dict() for k, v in self.records.items()}
1884 def as_json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:
1885 """
1886 Turn the data into a dict and then dump to JSON.
1887 """
1888 data = self.as_list()
1890 return as_json.encode(data, default=default, indent=indent, **kwargs)
1892 def json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:
1893 """
1894 Turn the data into a dict and then dump to JSON.
1895 """
1896 return self.as_json(default=default, indent=indent, **kwargs)
1898 def as_list(
1899 self,
1900 compact: bool = False,
1901 storage_to_dict: bool = False,
1902 datetime_to_str: bool = False,
1903 custom_types: list[type] = None,
1904 ) -> list[AnyDict]:
1905 """
1906 Get the data in a list of dicts.
1907 """
1908 if any([compact, storage_to_dict, datetime_to_str, custom_types]):
1909 return typing.cast(list[AnyDict], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types))
1911 return [_.as_dict() for _ in self.records.values()]
1913 def __getitem__(self, item: int) -> T_MetaInstance:
1914 """
1915 You can get a specific row by ID from a typedrows by using rows[idx] notation.
1917 Since pydal's implementation differs (they expect a list instead of a dict with id keys),
1918 using rows[0] will return the first row, regardless of its id.
1919 """
1920 try:
1921 return self.records[item]
1922 except KeyError as e:
1923 if item == 0 and (row := self.first()):
1924 # special case: pydal internals think Rows.records is a list, not a dict
1925 return row
1927 raise e
1929 def get(self, item: int) -> typing.Optional[T_MetaInstance]:
1930 """
1931 Get a row by ID, or receive None if it isn't in this result set.
1932 """
1933 return self.records.get(item)
1935 def update(self, **new_values: Any) -> bool:
1936 """
1937 Update the current rows in the database with new_values.
1938 """
1939 # cast to make mypy understand .id is a TypedField and not an int!
1940 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())
1942 ids = set(self.column("id"))
1943 query = table.id.belongs(ids)
1944 return bool(self.db(query).update(**new_values))
1946 def delete(self) -> bool:
1947 """
1948 Delete the currently selected rows from the database.
1949 """
1950 # cast to make mypy understand .id is a TypedField and not an int!
1951 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())
1953 ids = set(self.column("id"))
1954 query = table.id.belongs(ids)
1955 return bool(self.db(query).delete())
1957 def join(
1958 self,
1959 field: "Field | TypedField[Any]",
1960 name: str = None,
1961 constraint: Query = None,
1962 fields: list[str | Field] = None,
1963 orderby: Optional[str | Field] = None,
1964 ) -> T_MetaInstance:
1965 """
1966 This can be used to JOIN with some relationships after the initial select.
1968 Using the querybuilder's .join() method is prefered!
1969 """
1970 result = super().join(field, name, constraint, fields or [], orderby)
1971 return typing.cast(T_MetaInstance, result)
1973 def export_to_csv_file(
1974 self,
1975 ofile: typing.TextIO,
1976 null: Any = "<NULL>",
1977 delimiter: str = ",",
1978 quotechar: str = '"',
1979 quoting: int = csv.QUOTE_MINIMAL,
1980 represent: bool = False,
1981 colnames: list[str] = None,
1982 write_colnames: bool = True,
1983 *args: Any,
1984 **kwargs: Any,
1985 ) -> None:
1986 """
1987 Shadow export_to_csv_file from Rows, but with typing.
1989 See http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#Exporting-and-importing-data
1990 """
1991 super().export_to_csv_file(
1992 ofile,
1993 null,
1994 *args,
1995 delimiter=delimiter,
1996 quotechar=quotechar,
1997 quoting=quoting,
1998 represent=represent,
1999 colnames=colnames or self.colnames,
2000 write_colnames=write_colnames,
2001 **kwargs,
2002 )
2004 @classmethod
2005 def from_rows(
2006 cls, rows: Rows, model: typing.Type[T_MetaInstance], metadata: Metadata = None
2007 ) -> "TypedRows[T_MetaInstance]":
2008 """
2009 Internal method to convert a Rows object to a TypedRows.
2010 """
2011 return cls(rows, model, metadata=metadata)
2013 def __getstate__(self) -> AnyDict:
2014 """
2015 Used by dill to dump to bytes (exclude db connection etc).
2016 """
2017 return {
2018 "metadata": json.dumps(self.metadata, default=str),
2019 "records": self.records,
2020 "model": str(self.model._table),
2021 "colnames": self.colnames,
2022 }
2024 def __setstate__(self, state: AnyDict) -> None:
2025 """
2026 Used by dill when loading from a bytestring.
2027 """
2028 state["metadata"] = json.loads(state["metadata"])
2029 self.__dict__.update(state)
2030 # db etc. set after undill by caching.py
2033from .caching import ( # noqa: E402
2034 _remove_cache,
2035 _TypedalCache,
2036 _TypedalCacheDependency,
2037 create_and_hash_cache_key,
2038 get_expire,
2039 load_from_cache,
2040 save_to_cache,
2041)
2044class QueryBuilder(typing.Generic[T_MetaInstance]):
2045 """
2046 Abstration on top of pydal's query system.
2047 """
2049 model: typing.Type[T_MetaInstance]
2050 query: Query
2051 select_args: list[Any]
2052 select_kwargs: AnyDict
2053 relationships: dict[str, Relationship[Any]]
2054 metadata: Metadata
2056 def __init__(
2057 self,
2058 model: typing.Type[T_MetaInstance],
2059 add_query: Optional[Query] = None,
2060 select_args: Optional[list[Any]] = None,
2061 select_kwargs: Optional[AnyDict] = None,
2062 relationships: dict[str, Relationship[Any]] = None,
2063 metadata: Metadata = None,
2064 ):
2065 """
2066 Normally, you wouldn't manually initialize a QueryBuilder but start using a method on a TypedTable.
2068 Example:
2069 MyTable.where(...) -> QueryBuilder[MyTable]
2070 """
2071 self.model = model
2072 table = model._ensure_table_defined()
2073 default_query = typing.cast(Query, table.id > 0)
2074 self.query = add_query or default_query
2075 self.select_args = select_args or []
2076 self.select_kwargs = select_kwargs or {}
2077 self.relationships = relationships or {}
2078 self.metadata = metadata or {}
2080 def __str__(self) -> str:
2081 """
2082 Simple string representation for the query builder.
2083 """
2084 return f"QueryBuilder for {self.model}"
2086 def __repr__(self) -> str:
2087 """
2088 Advanced string representation for the query builder.
2089 """
2090 return (
2091 f"<QueryBuilder for {self.model} with "
2092 f"{len(self.select_args)} select args; "
2093 f"{len(self.select_kwargs)} select kwargs; "
2094 f"{len(self.relationships)} relationships; "
2095 f"query: {bool(self.query)}; "
2096 f"metadata: {self.metadata}; "
2097 f">"
2098 )
2100 def __bool__(self) -> bool:
2101 """
2102 Querybuilder is truthy if it has rows.
2103 """
2104 return self.count() > 0
2106 def _extend(
2107 self,
2108 add_query: Optional[Query] = None,
2109 overwrite_query: Optional[Query] = None,
2110 select_args: Optional[list[Any]] = None,
2111 select_kwargs: Optional[AnyDict] = None,
2112 relationships: dict[str, Relationship[Any]] = None,
2113 metadata: Metadata = None,
2114 ) -> "QueryBuilder[T_MetaInstance]":
2115 return QueryBuilder(
2116 self.model,
2117 (add_query & self.query) if add_query else overwrite_query or self.query,
2118 (self.select_args + select_args) if select_args else self.select_args,
2119 (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs,
2120 (self.relationships | relationships) if relationships else self.relationships,
2121 (self.metadata | (metadata or {})) if metadata else self.metadata,
2122 )
2124 def select(self, *fields: Any, **options: Any) -> "QueryBuilder[T_MetaInstance]":
2125 """
2126 Fields: database columns by name ('id'), by field reference (table.id) or other (e.g. table.ALL).
2128 Options:
2129 paraphrased from the web2py pydal docs,
2130 For more info, see http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache
2132 orderby: field(s) to order by. Supported:
2133 table.name - sort by name, ascending
2134 ~table.name - sort by name, descending
2135 <random> - sort randomly
2136 table.name|table.id - sort by two fields (first name, then id)
2138 groupby, having: together with orderby:
2139 groupby can be a field (e.g. table.name) to group records by
2140 having can be a query, only those `having` the condition are grouped
2142 limitby: tuple of min and max. When using the query builder, .paginate(limit, page) is recommended.
2143 distinct: bool/field. Only select rows that differ
2144 orderby_on_limitby (bool, default: True): by default, an implicit orderby is added when doing limitby.
2145 join: othertable.on(query) - do an INNER JOIN. Using TypeDAL relationships with .join() is recommended!
2146 left: othertable.on(query) - do a LEFT JOIN. Using TypeDAL relationships with .join() is recommended!
2147 cache: cache the query result to speed up repeated queries; e.g. (cache=(cache.ram, 3600), cacheable=True)
2148 """
2149 return self._extend(select_args=list(fields), select_kwargs=options)
2151 def where(
2152 self,
2153 *queries_or_lambdas: Query | typing.Callable[[typing.Type[T_MetaInstance]], Query],
2154 **filters: Any,
2155 ) -> "QueryBuilder[T_MetaInstance]":
2156 """
2157 Extend the builder's query.
2159 Can be used in multiple ways:
2160 .where(Query) -> with a direct query such as `Table.id == 5`
2161 .where(lambda table: table.id == 5) -> with a query via a lambda
2162 .where(id=5) -> via keyword arguments
2164 When using multiple where's, they will be ANDed:
2165 .where(lambda table: table.id == 5).where(lambda table: table.id == 6) == (table.id == 5) & (table.id=6)
2166 When passing multiple queries to a single .where, they will be ORed:
2167 .where(lambda table: table.id == 5, lambda table: table.id == 6) == (table.id == 5) | (table.id=6)
2168 """
2169 new_query = self.query
2170 table = self.model._ensure_table_defined()
2172 for field, value in filters.items():
2173 new_query &= table[field] == value
2175 subquery: DummyQuery | Query = DummyQuery()
2176 for query_or_lambda in queries_or_lambdas:
2177 if isinstance(query_or_lambda, _Query):
2178 subquery |= typing.cast(Query, query_or_lambda)
2179 elif callable(query_or_lambda):
2180 if result := query_or_lambda(self.model):
2181 subquery |= result
2182 elif isinstance(query_or_lambda, (Field, _Field)) or is_typed_field(query_or_lambda):
2183 subquery |= typing.cast(Query, query_or_lambda != None)
2184 else:
2185 raise ValueError(f"Unexpected query type ({type(query_or_lambda)}).")
2187 if subquery:
2188 new_query &= subquery
2190 return self._extend(overwrite_query=new_query)
2192 def join(
2193 self,
2194 *fields: str | typing.Type[TypedTable],
2195 method: JOIN_OPTIONS = None,
2196 on: OnQuery | list[Expression] | Expression = None,
2197 condition: Condition = None,
2198 ) -> "QueryBuilder[T_MetaInstance]":
2199 """
2200 Include relationship fields in the result.
2202 `fields` can be names of Relationships on the current model.
2203 If no fields are passed, all will be used.
2205 By default, the `method` defined in the relationship is used.
2206 This can be overwritten with the `method` keyword argument (left or inner)
2207 """
2208 # todo: allow limiting amount of related rows returned for join?
2210 relationships = self.model.get_relationships()
2212 if condition and on:
2213 raise ValueError("condition and on can not be used together!")
2214 elif condition:
2215 if len(fields) != 1:
2216 raise ValueError("join(field, condition=...) can only be used with exactly one field!")
2218 if isinstance(condition, pydal.objects.Query):
2219 condition = as_lambda(condition)
2221 relationships = {str(fields[0]): relationship(fields[0], condition=condition, join=method)}
2222 elif on:
2223 if len(fields) != 1:
2224 raise ValueError("join(field, on=...) can only be used with exactly one field!")
2226 if isinstance(on, pydal.objects.Expression):
2227 on = [on]
2229 if isinstance(on, list):
2230 on = as_lambda(on)
2231 relationships = {str(fields[0]): relationship(fields[0], on=on, join=method)}
2233 else:
2234 if fields:
2235 # join on every relationship
2236 relationships = {str(k): relationships[str(k)] for k in fields}
2238 if method:
2239 relationships = {str(k): r.clone(join=method) for k, r in relationships.items()}
2241 return self._extend(relationships=relationships)
2243 def cache(
2244 self, *deps: Any, expires_at: Optional[dt.datetime] = None, ttl: Optional[int | dt.timedelta] = None
2245 ) -> "QueryBuilder[T_MetaInstance]":
2246 """
2247 Enable caching for this query to load repeated calls from a dill row \
2248 instead of executing the sql and collecing matching rows again.
2249 """
2250 existing = self.metadata.get("cache", {})
2252 metadata: Metadata = {}
2254 cache_meta = typing.cast(
2255 CacheMetadata,
2256 self.metadata.get("cache", {})
2257 | {
2258 "enabled": True,
2259 "depends_on": existing.get("depends_on", []) + [str(_) for _ in deps],
2260 "expires_at": get_expire(expires_at=expires_at, ttl=ttl),
2261 },
2262 )
2264 metadata["cache"] = cache_meta
2265 return self._extend(metadata=metadata)
2267 def _get_db(self) -> TypeDAL:
2268 if db := self.model._db:
2269 return db
2270 else: # pragma: no cover
2271 raise EnvironmentError("@define or db.define is not called on this class yet!")
2273 def _select_arg_convert(self, arg: Any) -> Any:
2274 # typedfield are not really used at runtime anymore, but leave it in for safety:
2275 if isinstance(arg, TypedField): # pragma: no cover
2276 arg = arg._field
2278 return arg
2280 def delete(self) -> list[int]:
2281 """
2282 Based on the current query, delete rows and return a list of deleted IDs.
2283 """
2284 db = self._get_db()
2285 removed_ids = [_.id for _ in db(self.query).select("id")]
2286 if db(self.query).delete():
2287 # success!
2288 return removed_ids
2290 return []
2292 def _delete(self) -> str:
2293 db = self._get_db()
2294 return str(db(self.query)._delete())
2296 def update(self, **fields: Any) -> list[int]:
2297 """
2298 Based on the current query, update `fields` and return a list of updated IDs.
2299 """
2300 # todo: limit?
2301 db = self._get_db()
2302 updated_ids = db(self.query).select("id").column("id")
2303 if db(self.query).update(**fields):
2304 # success!
2305 return updated_ids
2307 return []
2309 def _update(self, **fields: Any) -> str:
2310 db = self._get_db()
2311 return str(db(self.query)._update(**fields))
2313 def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], AnyDict]:
2314 select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]
2315 select_kwargs = self.select_kwargs.copy()
2316 query = self.query
2317 model = self.model
2318 mut_metadata["query"] = query
2319 # require at least id of main table:
2320 select_fields = ", ".join([str(_) for _ in select_args])
2321 tablename = str(model)
2323 if add_id and f"{tablename}.id" not in select_fields:
2324 # fields of other selected, but required ID is missing.
2325 select_args.append(model.id)
2327 if self.relationships:
2328 query, select_args = self._handle_relationships_pre_select(query, select_args, select_kwargs, mut_metadata)
2330 return query, select_args, select_kwargs
2332 def to_sql(self, add_id: bool = False) -> str:
2333 """
2334 Generate the SQL for the built query.
2335 """
2336 db = self._get_db()
2338 query, select_args, select_kwargs = self._before_query({}, add_id=add_id)
2340 return str(db(query)._select(*select_args, **select_kwargs))
2342 def _collect(self) -> str:
2343 """
2344 Alias for to_sql, pydal-like syntax.
2345 """
2346 return self.to_sql()
2348 def _collect_cached(self, metadata: Metadata) -> "TypedRows[T_MetaInstance] | None":
2349 expires_at = metadata["cache"].get("expires_at")
2350 metadata["cache"] |= {
2351 # key is partly dependant on cache metadata but not these:
2352 "key": None,
2353 "status": None,
2354 "cached_at": None,
2355 "expires_at": None,
2356 }
2358 _, key = create_and_hash_cache_key(
2359 self.model,
2360 metadata,
2361 self.query,
2362 self.select_args,
2363 self.select_kwargs,
2364 self.relationships.keys(),
2365 )
2367 # re-set after creating key:
2368 metadata["cache"]["expires_at"] = expires_at
2369 metadata["cache"]["key"] = key
2371 return load_from_cache(key, self._get_db())
2373 def execute(self, add_id: bool = False) -> Rows:
2374 """
2375 Raw version of .collect which only executes the SQL, without performing any magic afterwards.
2376 """
2377 db = self._get_db()
2378 metadata = typing.cast(Metadata, self.metadata.copy())
2380 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)
2382 return db(query).select(*select_args, **select_kwargs)
2384 def collect(
2385 self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True
2386 ) -> "TypedRows[T_MetaInstance]":
2387 """
2388 Execute the built query and turn it into model instances, while handling relationships.
2389 """
2390 if _to is None:
2391 _to = TypedRows
2393 db = self._get_db()
2394 metadata = typing.cast(Metadata, self.metadata.copy())
2396 if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)):
2397 return result
2399 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)
2401 metadata["sql"] = db(query)._select(*select_args, **select_kwargs)
2403 if verbose: # pragma: no cover
2404 print(metadata["sql"])
2406 rows: Rows = db(query).select(*select_args, **select_kwargs)
2408 metadata["final_query"] = str(query)
2409 metadata["final_args"] = [str(_) for _ in select_args]
2410 metadata["final_kwargs"] = select_kwargs
2412 if verbose: # pragma: no cover
2413 print(rows)
2415 if not self.relationships:
2416 # easy
2417 typed_rows = _to.from_rows(rows, self.model, metadata=metadata)
2419 else:
2420 # harder: try to match rows to the belonging objects
2421 # assume structure of {'table': <data>} per row.
2422 # if that's not the case, return default behavior again
2423 typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to)
2425 # only saves if requested in metadata:
2426 return save_to_cache(typed_rows, rows)
2428 def _handle_relationships_pre_select(
2429 self,
2430 query: Query,
2431 select_args: list[Any],
2432 select_kwargs: AnyDict,
2433 metadata: Metadata,
2434 ) -> tuple[Query, list[Any]]:
2435 db = self._get_db()
2436 model = self.model
2438 metadata["relationships"] = set(self.relationships.keys())
2440 # query = self._update_query_for_inner(db, model, query)
2441 join = []
2442 for key, relation in self.relationships.items():
2443 if not relation.condition or relation.join != "inner":
2444 continue
2446 other = relation.get_table(db)
2447 other = other.with_alias(f"{key}_{hash(relation)}")
2448 join.append(other.on(relation.condition(model, other)))
2450 if limitby := select_kwargs.pop("limitby", None):
2451 # if limitby + relationships:
2452 # 1. get IDs of main table entries that match 'query'
2453 # 2. change query to .belongs(id)
2454 # 3. add joins etc
2456 kwargs = {"limitby": limitby}
2458 if join:
2459 kwargs["join"] = join
2461 ids = db(query)._select(model.id, **kwargs)
2462 query = model.id.belongs(ids)
2463 metadata["ids"] = ids
2465 if join:
2466 select_kwargs["join"] = join
2468 left = []
2470 for key, relation in self.relationships.items():
2471 other = relation.get_table(db)
2472 method: JOIN_OPTIONS = relation.join or DEFAULT_JOIN_OPTION
2474 select_fields = ", ".join([str(_) for _ in select_args])
2475 pre_alias = str(other)
2477 if f"{other}." not in select_fields:
2478 # no fields of other selected. add .ALL:
2479 select_args.append(other.ALL)
2480 elif f"{other}.id" not in select_fields:
2481 # fields of other selected, but required ID is missing.
2482 select_args.append(other.id)
2484 if relation.on:
2485 # if it has a .on, it's always a left join!
2486 on = relation.on(model, other)
2487 if not isinstance(on, list): # pragma: no cover
2488 on = [on]
2490 left.extend(on)
2491 elif method == "left":
2492 # .on not given, generate it:
2493 other = other.with_alias(f"{key}_{hash(relation)}")
2494 condition = typing.cast(Query, relation.condition(model, other))
2495 left.append(other.on(condition))
2496 else:
2497 # else: inner join (handled earlier)
2498 other = other.with_alias(f"{key}_{hash(relation)}") # only for replace
2499 # other = other.with_alias(f"{key}_{hash(relation)}")
2500 # query &= relation.condition(model, other)
2502 # if no fields of 'other' are included, add other.ALL
2503 # else: only add other.id if missing
2504 select_fields = ", ".join([str(_) for _ in select_args])
2506 post_alias = str(other).split(" AS ")[-1]
2507 if pre_alias != post_alias:
2508 # replace .select's with aliased:
2509 select_fields = select_fields.replace(
2510 f"{pre_alias}.",
2511 f"{post_alias}.",
2512 )
2514 select_args = select_fields.split(", ")
2516 select_kwargs["left"] = left
2517 return query, select_args
2519 def _collect_with_relationships(
2520 self, rows: Rows, metadata: Metadata, _to: typing.Type["TypedRows[Any]"]
2521 ) -> "TypedRows[T_MetaInstance]":
2522 """
2523 Transform the raw rows into Typed Table model instances.
2524 """
2525 db = self._get_db()
2526 main_table = self.model._ensure_table_defined()
2528 records = {}
2529 seen_relations: dict[str, set[str]] = defaultdict(set) # main id -> set of col + id for relation
2531 for row in rows:
2532 main = row[main_table]
2533 main_id = main.id
2535 if main_id not in records:
2536 records[main_id] = self.model(main)
2537 records[main_id]._with = list(self.relationships.keys())
2539 # setup up all relationship defaults (once)
2540 for col, relationship in self.relationships.items():
2541 records[main_id][col] = [] if relationship.multiple else None
2543 # now add other relationship data
2544 for column, relation in self.relationships.items():
2545 relationship_column = f"{column}_{hash(relation)}"
2547 # relationship_column works for aliases with the same target column.
2548 # if col + relationship not in the row, just use the regular name.
2550 relation_data = (
2551 row[relationship_column] if relationship_column in row else row[relation.get_table_name()]
2552 )
2554 if relation_data.id is None:
2555 # always skip None ids
2556 continue
2558 if f"{column}-{relation_data.id}" in seen_relations[main_id]:
2559 # speed up duplicates
2560 continue
2561 else:
2562 seen_relations[main_id].add(f"{column}-{relation_data.id}")
2564 relation_table = relation.get_table(db)
2565 # hopefully an instance of a typed table and a regular row otherwise:
2566 instance = relation_table(relation_data) if looks_like(relation_table, TypedTable) else relation_data
2568 if relation.multiple:
2569 # create list of T
2570 if not isinstance(records[main_id].get(column), list): # pragma: no cover
2571 # should already be set up before!
2572 setattr(records[main_id], column, [])
2574 records[main_id][column].append(instance)
2575 else:
2576 # create single T
2577 records[main_id][column] = instance
2579 return _to(rows, self.model, records, metadata=metadata)
2581 def collect_or_fail(self, exception: Exception = None) -> "TypedRows[T_MetaInstance]":
2582 """
2583 Call .collect() and raise an error if nothing found.
2585 Basically unwraps Optional type.
2586 """
2587 if result := self.collect():
2588 return result
2590 if not exception:
2591 exception = ValueError("Nothing found!")
2593 raise exception
2595 def __iter__(self) -> typing.Generator[T_MetaInstance, None, None]:
2596 """
2597 You can start iterating a Query Builder object before calling collect, for ease of use.
2598 """
2599 yield from self.collect()
2601 def count(self) -> int:
2602 """
2603 Return the amount of rows matching the current query.
2604 """
2605 db = self._get_db()
2606 model = self.model
2607 query = self.query
2609 for key, relation in self.relationships.items():
2610 if not relation.condition or relation.join != "inner":
2611 continue
2613 other = relation.get_table(db)
2614 other = other.with_alias(f"{key}_{hash(relation)}")
2615 query &= relation.condition(model, other)
2617 return db(query).count()
2619 def __paginate(
2620 self,
2621 limit: int,
2622 page: int = 1,
2623 ) -> "QueryBuilder[T_MetaInstance]":
2624 _from = limit * (page - 1)
2625 _to = limit * page
2627 available = self.count()
2629 metadata: Metadata = {}
2631 metadata["pagination"] = {
2632 "limit": limit,
2633 "current_page": page,
2634 "max_page": math.ceil(available / limit),
2635 "rows": available,
2636 "min_max": (_from, _to),
2637 }
2639 return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata)
2641 def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]":
2642 """
2643 Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset.
2645 Note: when using relationships, this limit is only applied to the 'main' table and any number of extra rows \
2646 can be loaded with relationship data!
2647 """
2648 builder = self.__paginate(limit, page)
2650 rows = typing.cast(PaginatedRows[T_MetaInstance], builder.collect(verbose=verbose, _to=PaginatedRows))
2652 rows._query_builder = builder
2653 return rows
2655 def _paginate(
2656 self,
2657 limit: int,
2658 page: int = 1,
2659 ) -> str:
2660 builder = self.__paginate(limit, page)
2661 return builder._collect()
2663 def chunk(self, chunk_size: int) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:
2664 """
2665 Generator that yields rows from a paginated source in chunks.
2667 This function retrieves rows from a paginated data source in chunks of the
2668 specified `chunk_size` and yields them as TypedRows.
2670 Example:
2671 ```
2672 for chunk_of_rows in Table.where(SomeTable.id > 5).chunk(100):
2673 for row in chunk_of_rows:
2674 # Process each row within the chunk.
2675 pass
2676 ```
2677 """
2678 page = 1
2680 while rows := self.__paginate(chunk_size, page).collect():
2681 yield rows
2682 page += 1
2684 def first(self, verbose: bool = False) -> T_MetaInstance | None:
2685 """
2686 Get the first row matching the currently built query.
2688 Also adds paginate, since it would be a waste to select more rows than needed.
2689 """
2690 if row := self.paginate(page=1, limit=1, verbose=verbose).first():
2691 return self.model.from_row(row)
2692 else:
2693 return None
2695 def _first(self) -> str:
2696 return self._paginate(page=1, limit=1)
2698 def first_or_fail(self, exception: Exception = None, verbose: bool = False) -> T_MetaInstance:
2699 """
2700 Call .first() and raise an error if nothing found.
2702 Basically unwraps Optional type.
2703 """
2704 if inst := self.first(verbose=verbose):
2705 return inst
2707 if not exception:
2708 exception = ValueError("Nothing found!")
2710 raise exception
2713S = typing.TypeVar("S")
2716class PaginatedRows(TypedRows[T_MetaInstance]):
2717 """
2718 Extension on top of rows that is used when calling .paginate() instead of .collect().
2719 """
2721 _query_builder: QueryBuilder[T_MetaInstance]
2723 @property
2724 def data(self) -> list[T_MetaInstance]:
2725 """
2726 Get the underlying data.
2727 """
2728 return list(self.records.values())
2730 @property
2731 def pagination(self) -> Pagination:
2732 """
2733 Get all page info.
2734 """
2735 pagination_data = self.metadata["pagination"]
2737 has_next_page = pagination_data["current_page"] < pagination_data["max_page"]
2738 has_prev_page = pagination_data["current_page"] > 1
2739 return {
2740 "total_items": pagination_data["rows"],
2741 "current_page": pagination_data["current_page"],
2742 "per_page": pagination_data["limit"],
2743 "total_pages": pagination_data["max_page"],
2744 "has_next_page": has_next_page,
2745 "has_prev_page": has_prev_page,
2746 "next_page": pagination_data["current_page"] + 1 if has_next_page else None,
2747 "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,
2748 }
2750 def next(self) -> Self:
2751 """
2752 Get the next page.
2753 """
2754 data = self.metadata["pagination"]
2755 if data["current_page"] >= data["max_page"]:
2756 raise StopIteration("Final Page")
2758 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)
2760 def previous(self) -> Self:
2761 """
2762 Get the previous page.
2763 """
2764 data = self.metadata["pagination"]
2765 if data["current_page"] <= 1:
2766 raise StopIteration("First Page")
2768 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)
2770 def as_dict(self, *_: Any, **__: Any) -> PaginateDict: # type: ignore
2771 """
2772 Convert to a dictionary with pagination info and original data.
2774 All arguments are ignored!
2775 """
2776 return {"data": super().as_dict(), "pagination": self.pagination}
2779class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover
2780 """
2781 Used to make pydal Set more typed.
2783 This class is not actually used, only 'cast' by TypeDAL.__call__
2784 """
2786 def count(self, distinct: bool = None, cache: AnyDict = None) -> int:
2787 """
2788 Count returns an int.
2789 """
2790 result = super().count(distinct, cache)
2791 return typing.cast(int, result)
2793 def select(self, *fields: Any, **attributes: Any) -> TypedRows[T_MetaInstance]:
2794 """
2795 Select returns a TypedRows of a user defined table.
2797 Example:
2798 result: TypedRows[MyTable] = db(MyTable.id > 0).select()
2800 for row in result:
2801 typing.reveal_type(row) # MyTable
2802 """
2803 rows = super().select(*fields, **attributes)
2804 return typing.cast(TypedRows[T_MetaInstance], rows)