Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/cardinal_pythonlib/colander_utils.py : 66%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# cardinal_pythonlib/colander_utils.py
4"""
5===============================================================================
7 Original code copyright (C) 2009-2021 Rudolf Cardinal (rudolf@pobox.com).
9 This file is part of cardinal_pythonlib.
11 Licensed under the Apache License, Version 2.0 (the "License");
12 you may not use this file except in compliance with the License.
13 You may obtain a copy of the License at
15 https://www.apache.org/licenses/LICENSE-2.0
17 Unless required by applicable law or agreed to in writing, software
18 distributed under the License is distributed on an "AS IS" BASIS,
19 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 See the License for the specific language governing permissions and
21 limitations under the License.
23===============================================================================
25**Functions for working with colander.**
27Colander: https://docs.pylonsproject.org/projects/colander/en/latest/
29"""
31import random
32from typing import (Any, Callable, Dict, Iterable, List, Optional,
33 Tuple, TYPE_CHECKING, Union)
35from cardinal_pythonlib.datetimefunc import (
36 coerce_to_pendulum,
37 PotentialDatetimeType,
38)
39from cardinal_pythonlib.logs import get_brace_style_log_with_null_handler
40# noinspection PyUnresolvedReferences
41import colander
42# noinspection PyUnresolvedReferences
43from colander import (
44 Boolean,
45 Date,
46 DateTime, # NB name clash with pendulum
47 Email,
48 Integer,
49 Invalid,
50 Length,
51 MappingSchema,
52 SchemaNode,
53 SchemaType,
54 String,
55)
56from deform.widget import (
57 CheckboxWidget,
58 DateTimeInputWidget,
59 HiddenWidget,
60)
61from pendulum import DateTime as Pendulum # NB name clash with colander
62from pendulum.parsing.exceptions import ParserError
64if TYPE_CHECKING:
65 # noinspection PyProtectedMember,PyUnresolvedReferences
66 from colander import _SchemaNode
68log = get_brace_style_log_with_null_handler(__name__)
70ColanderNullType = type(colander.null)
71ValidatorType = Callable[[SchemaNode, Any], None] # called as v(node, value)
73# =============================================================================
74# Debugging options
75# =============================================================================
77DEBUG_DANGER_VALIDATION = False
79if DEBUG_DANGER_VALIDATION:
80 log.warning("Debugging options enabled!")
82# =============================================================================
83# Constants
84# =============================================================================
86EMAIL_ADDRESS_MAX_LEN = 255 # https://en.wikipedia.org/wiki/Email_address
87SERIALIZED_NONE = "" # has to be a string; avoid "None" like the plague!
90# =============================================================================
91# New generic SchemaType classes
92# =============================================================================
94class PendulumType(SchemaType):
95 """
96 Colander :class:`SchemaType` for :class:`Pendulum` date/time objects.
97 """
98 def __init__(self, use_local_tz: bool = True):
99 self.use_local_tz = use_local_tz
100 super().__init__() # not necessary; SchemaType has no __init__
102 def serialize(self,
103 node: SchemaNode,
104 appstruct: Union[PotentialDatetimeType,
105 ColanderNullType]) \
106 -> Union[str, ColanderNullType]:
107 """
108 Serializes Python object to string representation.
109 """
110 if not appstruct:
111 return colander.null
112 try:
113 appstruct = coerce_to_pendulum(appstruct,
114 assume_local=self.use_local_tz)
115 except (ValueError, ParserError) as e:
116 raise Invalid(
117 node,
118 f"{appstruct!r} is not a pendulum.DateTime object; "
119 f"error was {e!r}")
120 return appstruct.isoformat()
122 def deserialize(self,
123 node: SchemaNode,
124 cstruct: Union[str, ColanderNullType]) \
125 -> Optional[Pendulum]:
126 """
127 Deserializes string representation to Python object.
128 """
129 if not cstruct:
130 return colander.null
131 try:
132 result = coerce_to_pendulum(cstruct,
133 assume_local=self.use_local_tz)
134 except (ValueError, ParserError) as e:
135 raise Invalid(node,
136 f"Invalid date/time: value={cstruct!r}, error={e!r}")
137 return result
140class AllowNoneType(SchemaType):
141 """
142 Serializes ``None`` to ``''``, and deserializes ``''`` to ``None``;
143 otherwise defers to the parent type.
145 A type which accepts serializing ``None`` to ``''`` and deserializing
146 ``''`` to ``None``. When the value is not equal to ``None``/``''``, it will
147 use (de)serialization of the given type. This can be used to make nodes
148 optional.
150 Example:
152 .. code-block:: python
154 date = colander.SchemaNode(
155 colander.NoneType(colander.DateTime()),
156 default=None,
157 missing=None,
158 )
160 NOTE ALSO that Colander nodes explicitly never validate a missing value;
161 see ``colander/__init__.py``, in :func:`_SchemaNode.deserialize`. We want
162 them to do so, essentially so we can pass in ``None`` to a form but have
163 the form refuse to validate if it's still ``None`` at submission.
165 """
166 def __init__(self, type_: SchemaType) -> None:
167 self.type_ = type_
169 def serialize(self, node: SchemaNode,
170 value: Any) -> Union[str, ColanderNullType]:
171 """
172 Serializes Python object to string representation.
173 """
174 if value is None:
175 retval = ''
176 else:
177 # noinspection PyUnresolvedReferences
178 retval = self.type_.serialize(node, value)
179 # log.debug("AllowNoneType.serialize: {!r} -> {!r}", value, retval)
180 return retval
182 def deserialize(self, node: SchemaNode,
183 value: Union[str, ColanderNullType]) -> Any:
184 """
185 Deserializes string representation to Python object.
186 """
187 if value is None or value == '':
188 retval = None
189 else:
190 # noinspection PyUnresolvedReferences
191 retval = self.type_.deserialize(node, value)
192 # log.debug("AllowNoneType.deserialize: {!r} -> {!r}", value, retval)
193 return retval
196# =============================================================================
197# Node helper functions
198# =============================================================================
200def get_values_and_permissible(values: Iterable[Tuple[Any, str]],
201 add_none: bool = False,
202 none_description: str = "[None]") \
203 -> Tuple[List[Tuple[Any, str]], List[Any]]:
204 """
205 Used when building Colander nodes.
207 Args:
208 values: an iterable of tuples like ``(value, description)`` used in
209 HTML forms
211 add_none: add a tuple ``(None, none_description)`` at the start of
212 ``values`` in the result?
214 none_description: the description used for ``None`` if ``add_none``
215 is set
217 Returns:
218 a tuple ``(values, permissible_values)``, where
220 - ``values`` is what was passed in (perhaps with the addition of the
221 "None" tuple at the start)
222 - ``permissible_values`` is a list of all the ``value`` elements of
223 the original ``values``
225 """
226 permissible_values = list(x[0] for x in values)
227 # ... does not include the None value; those do not go to the validator
228 if add_none:
229 none_tuple = (SERIALIZED_NONE, none_description)
230 values = [none_tuple] + list(values)
231 return values, permissible_values
234def get_child_node(parent: "_SchemaNode", child_name: str) -> "_SchemaNode":
235 """
236 Returns a child node from an instantiated :class:`colander.SchemaNode`
237 object. Such nodes are not accessible via ``self.mychild`` but must be
238 accessed via ``self.children``, which is a list of child nodes.
240 Args:
241 parent: the parent node object
242 child_name: the name of the child node
244 Returns:
245 the child node
247 Raises:
248 :exc:`StopIteration` if there isn't one
249 """
250 return next(c for c in parent.children if c.name == child_name)
253# =============================================================================
254# Validators
255# =============================================================================
257class EmailValidatorWithLengthConstraint(Email):
258 """
259 The Colander ``Email`` validator doesn't check length. This does.
260 """
261 def __init__(self, *args, min_length: int = 0, **kwargs) -> None:
262 self._length = Length(min_length, EMAIL_ADDRESS_MAX_LEN)
263 super().__init__(*args, **kwargs)
265 def __call__(self, node: SchemaNode, value: Any) -> None:
266 self._length(node, value)
267 super().__call__(node, value) # call Email regex validator
270# =============================================================================
271# Other new generic SchemaNode classes
272# =============================================================================
273# Note that we must pass both *args and **kwargs upwards, because SchemaNode
274# does some odd stuff with clone().
276# -----------------------------------------------------------------------------
277# Simple types
278# -----------------------------------------------------------------------------
280class OptionalIntNode(SchemaNode):
281 """
282 Colander node accepting integers but also blank values (i.e. it's
283 optional).
284 """
285 # YOU CANNOT USE ARGUMENTS THAT INFLUENCE THE STRUCTURE, because these Node
286 # objects get default-copied by Deform.
287 @staticmethod
288 def schema_type() -> SchemaType:
289 return AllowNoneType(Integer())
291 default = None
292 missing = None
295class OptionalStringNode(SchemaNode):
296 """
297 Colander node accepting strings but allowing them to be blank (optional).
299 Coerces None to ``""`` when serializing; otherwise it is coerced to
300 ``"None"``, i.e. a string literal containing the word "None", which is much
301 more wrong.
302 """
303 @staticmethod
304 def schema_type() -> SchemaType:
305 return AllowNoneType(String(allow_empty=True))
307 default = ""
308 missing = ""
311class MandatoryStringNode(SchemaNode):
312 """
313 Colander string node, where the string is obligatory.
315 CAVEAT: WHEN YOU PASS DATA INTO THE FORM, YOU MUST USE
317 .. code-block:: python
319 appstruct = {
320 somekey: somevalue or "",
321 # ^^^^^
322 # without this, None is converted to "None"
323 }
324 """
325 @staticmethod
326 def schema_type() -> SchemaType:
327 return String(allow_empty=False)
330class HiddenIntegerNode(OptionalIntNode):
331 """
332 Colander node containing an integer, that is hidden to the user.
333 """
334 widget = HiddenWidget()
337class HiddenStringNode(OptionalStringNode):
338 """
339 Colander node containing an optional string, that is hidden to the user.
340 """
341 widget = HiddenWidget()
344class BooleanNode(SchemaNode):
345 """
346 Colander node representing a boolean value with a checkbox widget.
347 """
348 schema_type = Boolean
349 widget = CheckboxWidget()
351 def __init__(self, *args, title: str = "?", label: str = "",
352 default: bool = False, **kwargs) -> None:
353 self.title = title # above the checkbox
354 self.label = label or title # to the right of the checkbox
355 self.default = default
356 self.missing = default
357 super().__init__(*args, **kwargs)
360# -----------------------------------------------------------------------------
361# Email addresses
362# -----------------------------------------------------------------------------
364class OptionalEmailNode(OptionalStringNode):
365 """
366 Colander string node, where the string can be blank but if not then it
367 must look like a valid e-mail address.
368 """
369 validator = EmailValidatorWithLengthConstraint()
372class MandatoryEmailNode(MandatoryStringNode):
373 """
374 Colander string node, requiring something that looks like a valid e-mail
375 address.
376 """
377 validator = EmailValidatorWithLengthConstraint()
380# -----------------------------------------------------------------------------
381# Date/time types
382# -----------------------------------------------------------------------------
384class DateTimeSelectorNode(SchemaNode):
385 """
386 Colander node containing a date/time.
387 """
388 schema_type = DateTime
389 missing = None
392class DateSelectorNode(SchemaNode):
393 """
394 Colander node containing a date.
395 """
396 schema_type = Date
397 missing = None
400DEFAULT_WIDGET_DATE_OPTIONS_FOR_PENDULUM = dict(
401 # http://amsul.ca/pickadate.js/date/#formatting-rules
402 format='yyyy-mm-dd',
403 selectMonths=True,
404 selectYears=True,
405)
406DEFAULT_WIDGET_TIME_OPTIONS_FOR_PENDULUM = dict(
407 # See http://amsul.ca/pickadate.js/time/#formatting-rules
408 # format='h:i A', # the default, e.g. "11:30 PM"
409 format='HH:i', # e.g. "23:30"
410 interval=30,
411)
414class OptionalPendulumNodeLocalTZ(SchemaNode):
415 """
416 Colander node containing an optional :class:`Pendulum` date/time, in which
417 the date/time is assumed to be in the local timezone.
418 """
419 @staticmethod
420 def schema_type() -> SchemaType:
421 return AllowNoneType(PendulumType(use_local_tz=True))
423 default = None
424 missing = None
425 widget = DateTimeInputWidget(
426 date_options=DEFAULT_WIDGET_DATE_OPTIONS_FOR_PENDULUM,
427 time_options=DEFAULT_WIDGET_TIME_OPTIONS_FOR_PENDULUM,
428 )
431OptionalPendulumNode = OptionalPendulumNodeLocalTZ # synonym for back-compatibility # noqa
434class OptionalPendulumNodeUTC(SchemaNode):
435 """
436 Colander node containing an optional :class:`Pendulum` date/time, in which
437 the date/time is assumed to be UTC.
438 """
439 @staticmethod
440 def schema_type() -> SchemaType:
441 return AllowNoneType(PendulumType(use_local_tz=False))
443 default = None
444 missing = None
445 widget = DateTimeInputWidget(
446 date_options=DEFAULT_WIDGET_DATE_OPTIONS_FOR_PENDULUM,
447 time_options=DEFAULT_WIDGET_TIME_OPTIONS_FOR_PENDULUM,
448 )
451# -----------------------------------------------------------------------------
452# Safety-checking nodes
453# -----------------------------------------------------------------------------
455class ValidateDangerousOperationNode(MappingSchema):
456 """
457 Colander node that can be added to forms allowing dangerous operations
458 (e.g. deletion of data). The node shows the user a code and requires the
459 user to type that code in, before it will permit the form to proceed.
461 For this to work, the containing form *must* inherit from
462 :class:`DynamicDescriptionsForm` with ``dynamic_descriptions=True``.
464 Usage is simple, like this:
466 .. code-block:: python
468 class AddSpecialNoteSchema(CSRFSchema):
469 table_name = HiddenStringNode()
470 server_pk = HiddenIntegerNode()
471 note = MandatoryStringNode(widget=TextAreaWidget(rows=20, cols=80))
472 danger = ValidateDangerousOperationNode()
474 """
475 target = HiddenStringNode()
476 user_entry = MandatoryStringNode(title="Validate this dangerous operation")
478 def __init__(self, *args, length: int = 4, allowed_chars: str = None,
479 **kwargs) -> None:
480 """
481 Args:
482 length: code length required from the user
483 allowed_chars: string containing the permitted characters
484 (by default, digits)
485 """
486 self.allowed_chars = allowed_chars or "0123456789"
487 self.length = length
488 super().__init__(*args, **kwargs)
490 # noinspection PyUnusedLocal
491 def after_bind(self, node: SchemaNode, kw: Dict[str, Any]) -> None:
492 # Accessing the nodes is fiddly!
493 target_node = get_child_node(self, "target")
494 # Also, this whole thing is a bit hard to get your head around.
495 # - This function will be called every time the form is accessed.
496 # - The first time (fresh form load), there will be no value in
497 # "target", so we set "target.default", and "target" will pick up
498 # that default value.
499 # - On subsequent times (e.g. form submission), there will be a value
500 # in "target", so the default is irrelevant.
501 # - This matters because we want "user_entry_node.description" to
502 # be correct.
503 # - Actually, easier is just to make "target" a static display?
504 # No; if you use widget=TextInputWidget(readonly=True), there is no
505 # form value rendered.
506 # - But it's hard to get the new value out of "target" at this point.
507 # - Should we do that in validate()?
508 # - No: on second rendering, after_bind() is called, and then
509 # validator() is called, but the visible form reflects changes made
510 # by after_bind() but NOT validator(); presumably Deform pulls the
511 # contents in between those two. Hmm.
512 # - Particularly "hmm" as we don't have access to form data at the
513 # point of after_bind().
514 # - The problem is probably that deform.field.Field.__init__ copies its
515 # schema.description. Yes, that's the problem.
516 # - So: a third option: a display value (which we won't get back) as
517 # well as a hidden value that we will? No, no success.
518 # - Or a fourth: something whose "description" is a property, not a
519 # str? No -- when you copy a property, you copy the value not the
520 # function.
521 # - Fifthly: a new DangerValidationForm that rewrites its field
522 # descriptions after validation. That works!
523 target_value = ''.join(random.choice(self.allowed_chars)
524 for i in range(self.length))
525 target_node.default = target_value
526 # Set the description:
527 if DEBUG_DANGER_VALIDATION:
528 log.debug("after_bind: setting description to {!r}", target_value)
529 self.set_description(target_value)
530 # ... may be overridden immediately by validator() if this is NOT the
531 # first rendering
533 def validator(self, node: SchemaNode, value: Any) -> None:
534 user_entry_value = value['user_entry']
535 target_value = value['target']
536 # Set the description:
537 if DEBUG_DANGER_VALIDATION:
538 log.debug("validator: setting description to {!r}", target_value)
539 self.set_description(target_value)
540 # arse!
541 value['display_target'] = target_value
542 # Check the value
543 if user_entry_value != target_value:
544 raise Invalid(
545 node,
546 f"Not correctly validated "
547 f"(user_entry_value={user_entry_value!r}, "
548 f"target_value={target_value!r}")
550 def set_description(self, target_value: str) -> None:
551 user_entry_node = get_child_node(self, "user_entry")
552 prefix = "Please enter the following: "
553 user_entry_node.description = prefix + target_value
554 if DEBUG_DANGER_VALIDATION:
555 log.debug("user_entry_node.description: {!r}",
556 user_entry_node.description)