Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/chameleon/compiler.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import re
2import sys
3import itertools
4import logging
5import threading
6import functools
7import collections
8import pickle
9import textwrap
11from .astutil import load
12from .astutil import store
13from .astutil import param
14from .astutil import swap
15from .astutil import subscript
16from .astutil import node_annotations
17from .astutil import annotated
18from .astutil import NameLookupRewriteVisitor
19from .astutil import Comment
20from .astutil import Symbol
21from .astutil import Builtin
22from .astutil import Static
23from .astutil import TokenRef
24from .astutil import Node
26from .codegen import TemplateCodeGenerator
27from .codegen import template
29from .tal import ErrorInfo
30from .tal import NAME
31from .i18n import simple_translate
33from .nodes import Text
34from .nodes import Value
35from .nodes import Substitution
36from .nodes import Assignment
37from .nodes import Module
38from .nodes import Context
39from .nodes import Is
40from .nodes import IsNot
41from .nodes import Equals
42from .nodes import Logical
43from .nodes import And
45from .tokenize import Token
46from .config import DEBUG_MODE
47from .exc import TranslationError
48from .exc import ExpressionError
49from .parser import groupdict
51from .utils import DebuggingOutputStream
52from .utils import char2entity
53from .utils import ListDictProxy
54from .utils import native_string
55from .utils import byte_string
56from .utils import string_type
57from .utils import unicode_string
58from .utils import version
59from .utils import ast
60from .utils import safe_native
61from .utils import builtins
62from .utils import decode_htmlentities
63from .utils import join
65if version >= (3, 0, 0):
66 long = int
68log = logging.getLogger('chameleon.compiler')
70COMPILER_INTERNALS_OR_DISALLOWED = set([
71 "econtext",
72 "rcontext",
73 "str",
74 "int",
75 "float",
76 "long",
77 "len",
78 "None",
79 "True",
80 "False",
81 "RuntimeError",
82 ])
85RE_MANGLE = re.compile(r'[^\w_]')
86RE_NAME = re.compile('^%s$' % NAME)
88if DEBUG_MODE:
89 LIST = template("cls()", cls=DebuggingOutputStream, mode="eval")
90else:
91 LIST = template("[]", mode="eval")
94def identifier(prefix, suffix=None):
95 return "__%s_%s" % (prefix, mangle(suffix or id(prefix)))
98def mangle(string):
99 return RE_MANGLE.sub(
100 '_', unicode_string(string)
101 ).replace('\n', '').replace('-', '_')
104def load_econtext(name):
105 return template("getitem(KEY)", KEY=ast.Str(s=name), mode="eval")
108def store_econtext(name):
109 name = native_string(name)
110 return subscript(name, load("econtext"), ast.Store())
113def store_rcontext(name):
114 name = native_string(name)
115 return subscript(name, load("rcontext"), ast.Store())
118def set_token(stmts, token):
119 pos = getattr(token, "pos", 0)
120 body = template("__token = pos", pos=TokenRef(pos, len(token)))
121 return body + stmts
124def eval_token(token):
125 try:
126 line, column = token.location
127 filename = token.filename
128 except AttributeError:
129 line, column = 0, 0
130 filename = "<string>"
132 string = safe_native(token)
134 return template(
135 "(string, line, col)",
136 string=ast.Str(s=string),
137 line=ast.Num(n=line),
138 col=ast.Num(n=column),
139 mode="eval"
140 )
143emit_node_if_non_trivial = template(is_func=True, func_args=('node',),
144 source=r"""
145 if node is not None:
146 __append(node)
147""")
150emit_bool = template(is_func=True,
151 func_args=('target', 's', 'default_marker', 'default'),
152 func_defaults=(None, None), source=r"""
153 if target is default_marker:
154 target = default
155 elif target:
156 target = s
157 else:
158 target = None""")
161emit_convert = template(is_func=True,
162 func_args=('target', 'encoded', 'str', 'long', 'type',
163 'default_marker', 'default'),
164 func_defaults=(byte_string, unicode_string, long, type,
165 None),
166 source=r"""
167 if target is None:
168 pass
169 elif target is default_marker:
170 target = default
171 else:
172 __tt = type(target)
174 if __tt is int or __tt is float or __tt is long:
175 target = str(target)
176 elif __tt is encoded:
177 target = decode(target)
178 elif __tt is not str:
179 try:
180 target = target.__html__
181 except AttributeError:
182 __converted = convert(target)
183 target = str(target) if target is __converted else __converted
184 else:
185 target = target()""")
188emit_func_convert = template(is_func=True,
189 func_args=('func', 'encoded', 'str','long','type'),
190 func_defaults=(byte_string, unicode_string, long,
191 type),
192 source=r"""
193 def func(target):
194 if target is None:
195 return
197 __tt = type(target)
199 if __tt is int or __tt is float or __tt is long:
200 target = str(target)
202 elif __tt is encoded:
203 target = decode(target)
205 elif __tt is not str:
206 try:
207 target = target.__html__
208 except AttributeError:
209 __converted = convert(target)
210 target = str(target) if target is __converted else __converted
211 else:
212 target = target()
214 return target""")
217emit_translate = template(is_func=True,
218 func_args=('target', 'msgid', 'target_language',
219 'default'),
220 func_defaults=(None,),
221 source=r"""
222 target = translate(msgid, default=default, domain=__i18n_domain,
223 context=__i18n_context,
224 target_language=target_language)""")
227emit_func_convert_and_escape = template(
228 is_func=True,
229 func_args=('func', 'str', 'long', 'type', 'encoded'),
230 func_defaults=(unicode_string, long, type, byte_string,),
231 source=r"""
232 def func(target, quote, quote_entity, default, default_marker):
233 if target is None:
234 return
236 if target is default_marker:
237 return default
239 __tt = type(target)
241 if __tt is int or __tt is float or __tt is long:
242 target = str(target)
243 else:
244 if __tt is encoded:
245 target = decode(target)
246 elif __tt is not str:
247 try:
248 target = target.__html__
249 except:
250 __converted = convert(target)
251 target = str(target) if target is __converted \
252 else __converted
253 else:
254 return target()
256 if target is not None:
257 try:
258 escape = __re_needs_escape(target) is not None
259 except TypeError:
260 pass
261 else:
262 if escape:
263 # Character escape
264 if '&' in target:
265 target = target.replace('&', '&')
266 if '<' in target:
267 target = target.replace('<', '<')
268 if '>' in target:
269 target = target.replace('>', '>')
270 if quote is not None and quote in target:
271 target = target.replace(quote, quote_entity)
273 return target""")
276class EmitText(Node):
277 """Append text to output."""
279 _fields = "s",
282class Scope(Node):
283 """"Set a local output scope."""
285 _fields = "body", "append", "stream"
287 body = None
288 append = None
289 stream = None
292class Interpolator(object):
293 braces_required_regex = re.compile(
294 r'(\$)?\$({(?P<expression>.*)})',
295 re.DOTALL)
297 braces_optional_regex = re.compile(
298 r'(\$)?\$({(?P<expression>.*)}|(?P<variable>[A-Za-z][A-Za-z0-9_]*))',
299 re.DOTALL)
301 def __init__(self, expression, braces_required, translate=False,
302 decode_htmlentities=False):
303 self.expression = expression
304 self.regex = self.braces_required_regex if braces_required else \
305 self.braces_optional_regex
306 self.translate = translate
307 self.decode_htmlentities = decode_htmlentities
309 def __call__(self, name, engine):
310 """The strategy is to find possible expression strings and
311 call the ``validate`` function of the parser to validate.
313 For every possible starting point, the longest possible
314 expression is tried first, then the second longest and so
315 forth.
317 Example 1:
319 ${'expressions use the ${<expression>} format'}
321 The entire expression is attempted first and it is also the
322 only one that validates.
324 Example 2:
326 ${'Hello'} ${'world!'}
328 Validation of the longest possible expression (the entire
329 string) will fail, while the second round of attempts,
330 ``${'Hello'}`` and ``${'world!'}`` respectively, validate.
332 """
334 body = []
335 nodes = []
336 text = self.expression
338 expr_map = {}
339 translate = self.translate
341 while text:
342 matched = text
343 m = self.regex.search(matched)
344 if m is None:
345 text = text.replace('$$', '$')
346 nodes.append(ast.Str(s=text))
347 break
349 part = text[:m.start()]
350 text = text[m.start():]
352 skip = text.startswith('$$')
353 if skip:
354 part = part + '$'
356 if part:
357 part = part.replace('$$', '$')
358 node = ast.Str(s=part)
359 nodes.append(node)
361 if skip:
362 text = text[2:]
363 continue
365 if not body:
366 target = name
367 else:
368 target = store("%s_%d" % (name.id, text.pos))
370 while True:
371 d = groupdict(m, matched)
372 string = d["expression"] or d.get("variable") or ""
374 if self.decode_htmlentities:
375 string = decode_htmlentities(string)
377 if string:
378 try:
379 compiler = engine.parse(string)
380 body += compiler.assign_text(target)
381 except ExpressionError:
382 matched = matched[m.start():m.end() - 1]
383 m = self.regex.search(matched)
384 if m is None:
385 raise
387 continue
388 else:
389 s = m.group()
390 assign = ast.Assign(targets=[target], value=ast.Str(s=s))
391 body += [assign]
393 break
395 # If one or more expressions are not simple names, we
396 # disable translation.
397 if RE_NAME.match(string) is None:
398 translate = False
400 # if this is the first expression, use the provided
401 # assignment name; otherwise, generate one (here based
402 # on the string position)
403 node = load(target.id)
404 nodes.append(node)
406 expr_map[node] = safe_native(string)
408 text = text[len(m.group()):]
410 if len(nodes) == 1:
411 target = nodes[0]
413 if translate and isinstance(target, ast.Str):
414 target = template(
415 "translate(msgid, domain=__i18n_domain, context=__i18n_context, target_language=target_language)",
416 msgid=target, mode="eval",
417 target_language=load("target_language"),
418 )
419 else:
420 if translate:
421 formatting_string = ""
422 keys = []
423 values = []
425 for node in nodes:
426 if isinstance(node, ast.Str):
427 formatting_string += node.s
428 else:
429 string = expr_map[node]
430 formatting_string += "${%s}" % string
431 keys.append(ast.Str(s=string))
432 values.append(node)
434 target = template(
435 "translate(msgid, mapping=mapping, domain=__i18n_domain, context=__i18n_context, target_language=target_language)",
436 msgid=ast.Str(s=formatting_string),
437 target_language=load("target_language"),
438 mapping=ast.Dict(keys=keys, values=values),
439 mode="eval"
440 )
441 else:
442 nodes = [
443 node if isinstance(node, ast.Str) else
444 template(
445 "NODE if NODE is not None else ''",
446 NODE=node, mode="eval"
447 )
448 for node in nodes
449 ]
451 target = ast.BinOp(
452 left=ast.Str(s="%s" * len(nodes)),
453 op=ast.Mod(),
454 right=ast.Tuple(elts=nodes, ctx=ast.Load()))
456 body += [ast.Assign(targets=[name], value=target)]
457 return body
460class ExpressionEngine(object):
461 """Expression engine.
463 This test demonstrates how to configure and invoke the engine.
465 >>> from chameleon import tales
466 >>> parser = tales.ExpressionParser({
467 ... 'python': tales.PythonExpr,
468 ... 'not': tales.NotExpr,
469 ... 'exists': tales.ExistsExpr,
470 ... 'string': tales.StringExpr,
471 ... }, 'python')
473 >>> engine = ExpressionEngine(parser)
475 An expression evaluation function:
477 >>> eval = lambda expression: tales.test(
478 ... tales.IdentityExpr(expression), engine)
480 We have provided 'python' as the default expression type. This
481 means that when no prefix is given, the expression is evaluated as
482 a Python expression:
484 >>> eval('not False')
485 True
487 Note that the ``type`` prefixes bind left. If ``not`` and
488 ``exits`` are two expression type prefixes, consider the
489 following::
491 >>> eval('not: exists: int(None)')
492 True
494 The pipe operator binds right. In the following example, but
495 arguments are evaluated against ``not: exists: ``.
497 >>> eval('not: exists: help')
498 False
499 """
501 supported_char_escape_set = set(('&', '<', '>'))
503 def __init__(self, parser, char_escape=(),
504 default=None, default_marker=None):
505 self._parser = parser
506 self._char_escape = char_escape
507 self._default = default
508 self._default_marker = default_marker
510 def __call__(self, string, target):
511 # BBB: This method is deprecated. Instead, a call should first
512 # be made to ``parse`` and then one of the assignment methods
513 # ("value" or "text").
515 compiler = self.parse(string)
516 return compiler(string, target)
518 def parse(self, string, handle_errors=True, char_escape=None):
519 expression = self._parser(string)
520 compiler = self.get_compiler(expression, string, handle_errors, char_escape)
521 return ExpressionCompiler(compiler, self)
523 def get_compiler(self, expression, string, handle_errors, char_escape):
524 if char_escape is None:
525 char_escape = self._char_escape
526 def compiler(target, engine, result_type=None, *args):
527 stmts = expression(target, engine)
529 if result_type is not None:
530 method = getattr(self, '_convert_%s' % result_type)
531 steps = method(target, char_escape, *args)
532 stmts.extend(steps)
534 if handle_errors:
535 return set_token(stmts, string.strip())
537 return stmts
539 return compiler
541 def _convert_bool(self, target, char_escape, s):
542 """Converts value given by ``target`` to a string ``s`` if the
543 target is a true value, otherwise ``None``.
544 """
546 return emit_bool(
547 target, ast.Str(s=s),
548 default=self._default,
549 default_marker=self._default_marker
550 )
552 def _convert_structure(self, target, char_escape):
553 """Converts value given by ``target`` to structure output."""
555 return emit_convert(
556 target,
557 default=self._default,
558 default_marker=self._default_marker,
559 )
561 def _convert_text(self, target, char_escape):
562 """Converts value given by ``target`` to text."""
564 if not char_escape:
565 return self._convert_structure(target, char_escape)
567 # This is a cop-out - we really only support a very select
568 # set of escape characters
569 other = set(char_escape) - self.supported_char_escape_set
571 if other:
572 for supported in '"', '\'', '':
573 if supported in char_escape:
574 quote = supported
575 break
576 else:
577 raise RuntimeError(
578 "Unsupported escape set: %s." % repr(char_escape)
579 )
580 else:
581 quote = '\0'
583 entity = char2entity(quote or '\0')
585 return template(
586 "TARGET = __quote(TARGET, QUOTE, Q_ENTITY, DEFAULT, MARKER)",
587 TARGET=target,
588 QUOTE=ast.Str(s=quote),
589 Q_ENTITY=ast.Str(s=entity),
590 DEFAULT=self._default,
591 MARKER=self._default_marker,
592 )
595class ExpressionCompiler(object):
596 def __init__(self, compiler, engine):
597 self.compiler = compiler
598 self.engine = engine
600 def assign_bool(self, target, s):
601 return self.compiler(target, self.engine, "bool", s)
603 def assign_text(self, target):
604 return self.compiler(target, self.engine, "text")
606 def assign_value(self, target):
607 return self.compiler(target, self.engine)
610class ExpressionEvaluator(object):
611 """Evaluates dynamic expression.
613 This is not particularly efficient, but supported for legacy
614 applications.
616 >>> from chameleon import tales
617 >>> parser = tales.ExpressionParser({'python': tales.PythonExpr}, 'python')
618 >>> engine = functools.partial(ExpressionEngine, parser)
620 >>> evaluate = ExpressionEvaluator(engine, {
621 ... 'foo': 'bar',
622 ... })
624 The evaluation function is passed the local and remote context,
625 the expression type and finally the expression.
627 >>> evaluate({'boo': 'baz'}, {}, 'python', 'foo + boo')
628 'barbaz'
630 The cache is now primed:
632 >>> evaluate({'boo': 'baz'}, {}, 'python', 'foo + boo')
633 'barbaz'
635 Note that the call method supports currying of the expression
636 argument:
638 >>> python = evaluate({'boo': 'baz'}, {}, 'python')
639 >>> python('foo + boo')
640 'barbaz'
642 """
644 __slots__ = "_engine", "_cache", "_names", "_builtins"
646 def __init__(self, engine, builtins):
647 self._engine = engine
648 self._names, self._builtins = zip(*builtins.items())
649 self._cache = {}
651 def __call__(self, econtext, rcontext, expression_type, string=None):
652 if string is None:
653 return functools.partial(
654 self.__call__, econtext, rcontext, expression_type
655 )
657 expression = "%s:%s" % (expression_type, string)
659 try:
660 evaluate = self._cache[expression]
661 except KeyError:
662 assignment = Assignment(["_result"], expression, True)
663 module = Module("evaluate", Context(assignment))
665 compiler = Compiler(
666 self._engine, module, "<string>", string,
667 ('econtext', 'rcontext') + self._names
668 )
670 env = {}
671 exec(compiler.code, env)
672 evaluate = self._cache[expression] = env["evaluate"]
674 evaluate(econtext, rcontext, *self._builtins)
675 return econtext['_result']
678class NameTransform(object):
679 """
680 >>> nt = NameTransform(
681 ... set(('foo', 'bar', )), {'boo': 'boz'},
682 ... ('econtext', ),
683 ... )
685 >>> def test(node):
686 ... rewritten = nt(node)
687 ... module = ast.Module([ast.fix_missing_locations(rewritten)])
688 ... codegen = TemplateCodeGenerator(module)
689 ... return codegen.code
691 Any odd name:
693 >>> test(load('frobnitz'))
694 "getitem('frobnitz')"
696 A 'builtin' name will first be looked up via ``get`` allowing fall
697 back to the global builtin value:
699 >>> test(load('foo'))
700 "get('foo', foo)"
702 Internal names (with two leading underscores) are left alone:
704 >>> test(load('__internal'))
705 '__internal'
707 Compiler internals or disallowed names:
709 >>> test(load('econtext'))
710 'econtext'
712 Aliased names:
714 >>> test(load('boo'))
715 'boz'
717 """
719 def __init__(self, builtins, aliases, internals):
720 self.builtins = builtins
721 self.aliases = aliases
722 self.internals = internals
724 def __call__(self, node):
725 name = node.id
727 # Don't rewrite names that begin with an underscore; they are
728 # internal and can be assumed to be locally defined. This
729 # policy really should be part of the template program, not
730 # defined here in the compiler.
731 if name.startswith('__') or name in self.internals:
732 return node
734 if isinstance(node.ctx, ast.Store):
735 return store_econtext(name)
737 aliased = self.aliases.get(name)
738 if aliased is not None:
739 return load(aliased)
741 # If the name is a Python global, first try acquiring it from
742 # the dynamic context, then fall back to the global.
743 if name in self.builtins:
744 return template(
745 "get(key, name)",
746 mode="eval",
747 key=ast.Str(s=name),
748 name=load(name),
749 )
751 # Otherwise, simply acquire it from the dynamic context.
752 return load_econtext(name)
755class ExpressionTransform(object):
756 """Internal wrapper to transform expression nodes into assignment
757 statements.
759 The node input may use the provided expression engine, but other
760 expression node types are supported such as ``Builtin`` which
761 simply resolves a built-in name.
763 Used internally be the compiler.
764 """
766 loads_symbol = Symbol(pickle.loads)
768 def __init__(self, engine_factory, cache, visitor, strict=True):
769 self.engine_factory = engine_factory
770 self.cache = cache
771 self.strict = strict
772 self.visitor = visitor
774 def __call__(self, expression, target):
775 if isinstance(target, string_type):
776 target = store(target)
778 try:
779 stmts = self.translate(expression, target)
780 except ExpressionError:
781 if self.strict:
782 raise
784 exc = sys.exc_info()[1]
785 p = pickle.dumps(exc, -1)
787 stmts = template(
788 "__exc = loads(p)", loads=self.loads_symbol, p=ast.Str(s=p)
789 )
791 stmts += set_token([ast.Raise(exc=load("__exc"))], exc.token)
793 # Apply visitor to each statement
794 for stmt in stmts:
795 self.visitor(stmt)
797 return stmts
799 def translate(self, expression, target):
800 if isinstance(target, string_type):
801 target = store(target)
803 cached = self.cache.get(expression)
805 if cached is not None:
806 stmts = [ast.Assign(targets=[target], value=cached)]
807 elif isinstance(expression, ast.expr):
808 stmts = [ast.Assign(targets=[target], value=expression)]
809 else:
810 # The engine interface supports simple strings, which
811 # default to expression nodes
812 if isinstance(expression, string_type):
813 expression = Value(expression, True)
815 kind = type(expression).__name__
816 visitor = getattr(self, "visit_%s" % kind)
817 stmts = visitor(expression, target)
819 # Add comment
820 target_id = getattr(target, "id", target)
821 comment = Comment(" %r -> %s" % (expression, target_id))
822 stmts.insert(0, comment)
824 return stmts
826 def visit_Value(self, node, target):
827 engine = self.engine_factory(
828 default=node.default,
829 default_marker=node.default_marker
830 )
831 compiler = engine.parse(node.value)
832 return compiler.assign_value(target)
834 def visit_Copy(self, node, target):
835 return self.translate(node.expression, target)
837 def visit_Substitution(self, node, target):
838 engine = self.engine_factory(
839 default=node.default,
840 default_marker=node.default_marker
841 )
842 compiler = engine.parse(node.value, char_escape=node.char_escape)
843 return compiler.assign_text(target)
845 def visit_Negate(self, node, target):
846 return self.translate(node.value, target) + \
847 template("TARGET = not TARGET", TARGET=target)
849 def visit_BinOp(self, node, target):
850 expression = self.translate(node.left, "__expression")
851 value = self.translate(node.right, "__value")
853 op = {
854 Is: "is",
855 IsNot: "is not",
856 Equals: "==",
857 }[node.op]
858 return expression + value + \
859 template("TARGET = __expression %s __value" % op, TARGET=target)
861 def visit_Boolean(self, node, target):
862 engine = self.engine_factory(
863 default=node.default,
864 default_marker=node.default_marker,
865 )
866 compiler = engine.parse(node.value)
867 return compiler.assign_bool(target, node.s)
869 def visit_Interpolation(self, node, target):
870 expr = node.value
871 if isinstance(expr, Substitution):
872 engine = self.engine_factory(
873 char_escape=expr.char_escape,
874 default=expr.default,
875 default_marker=expr.default_marker
876 )
877 elif isinstance(expr, Value):
878 engine = self.engine_factory(
879 default=expr.default,
880 default_marker=expr.default_marker
881 )
882 else:
883 raise RuntimeError("Bad value: %r." % node.value)
885 interpolator = Interpolator(
886 expr.value, node.braces_required,
887 translate=node.translation,
888 decode_htmlentities=True
889 )
891 compiler = engine.get_compiler(
892 interpolator, expr.value, True, ()
893 )
894 return compiler(target, engine, "text")
896 def visit_Translate(self, node, target):
897 if node.msgid is not None:
898 msgid = ast.Str(s=node.msgid)
899 else:
900 msgid = target
901 return self.translate(node.node, target) + \
902 emit_translate(
903 target, msgid, "target_language",
904 default=target
905 )
907 def visit_Static(self, node, target):
908 value = annotated(node)
909 return [ast.Assign(targets=[target], value=value)]
911 def visit_Builtin(self, node, target):
912 value = annotated(node)
913 return [ast.Assign(targets=[target], value=value)]
915 def visit_Symbol(self, node, target):
916 value = annotated(node)
917 return template("TARGET = SYMBOL", TARGET=target, SYMBOL=node)
920class Compiler(object):
921 """Generic compiler class.
923 Iterates through nodes and yields Python statements which form a
924 template program.
925 """
927 exceptions = NameError, \
928 ValueError, \
929 AttributeError, \
930 LookupError, \
931 TypeError
933 defaults = {
934 'translate': Symbol(simple_translate),
935 'decode': Builtin("str"),
936 'convert': Builtin("str"),
937 'on_error_handler': Builtin("str")
938 }
940 lock = threading.Lock()
942 global_builtins = set(builtins.__dict__)
944 def __init__(self, engine_factory, node, filename, source,
945 builtins={}, strict=True):
946 self._scopes = [set()]
947 self._expression_cache = {}
948 self._translations = []
949 self._builtins = builtins
950 self._aliases = [{}]
951 self._macros = []
952 self._current_slot = []
954 internals = COMPILER_INTERNALS_OR_DISALLOWED | \
955 set(self.defaults)
957 transform = NameTransform(
958 self.global_builtins | set(builtins),
959 ListDictProxy(self._aliases),
960 internals,
961 )
963 self._visitor = visitor = NameLookupRewriteVisitor(transform)
965 self._engine = ExpressionTransform(
966 engine_factory,
967 self._expression_cache,
968 visitor,
969 strict=strict,
970 )
972 if isinstance(node_annotations, dict):
973 self.lock.acquire()
974 backup = node_annotations.copy()
975 else:
976 backup = None
978 try:
979 module = ast.Module([])
980 module.body += self.visit(node)
981 ast.fix_missing_locations(module)
983 class Generator(TemplateCodeGenerator):
984 scopes = [Scope()]
986 def visit_EmitText(self, node):
987 append = load(self.scopes[-1].append or "__append")
988 for node in template("append(s)", append=append, s=ast.Str(s=node.s)):
989 self.visit(node)
991 def visit_Scope(self, node):
992 self.scopes.append(node)
993 body = list(node.body)
994 swap(body, load(node.append), "__append")
995 if node.stream:
996 swap(body, load(node.stream), "__stream")
997 for node in body:
998 self.visit(node)
999 self.scopes.pop()
1001 generator = Generator(module, source)
1002 tokens = [
1003 Token(source[pos:pos + length], pos, source)
1004 for pos, length in generator.tokens
1005 ]
1006 token_map_def = "__tokens = {" + ", ".join("%d: %r" % (
1007 token.pos,
1008 (token, ) + token.location
1009 ) for token in tokens) + "}"
1010 finally:
1011 if backup is not None:
1012 node_annotations.clear()
1013 node_annotations.update(backup)
1014 self.lock.release()
1016 self.code = "\n".join((
1017 "__filename = %r\n" % filename,
1018 token_map_def,
1019 generator.code
1020 ))
1022 def visit(self, node):
1023 if node is None:
1024 return ()
1025 kind = type(node).__name__
1026 visitor = getattr(self, "visit_%s" % kind)
1027 iterator = visitor(node)
1028 result = []
1029 for key, group in itertools.groupby(iterator, lambda node: node.__class__):
1030 nodes = list(group)
1031 if key is EmitText:
1032 text = join(node.s for node in nodes)
1033 nodes = [EmitText(text)]
1034 result.extend(nodes)
1035 return result
1038 def visit_Sequence(self, node):
1039 for item in node.items:
1040 for stmt in self.visit(item):
1041 yield stmt
1043 def visit_Element(self, node):
1044 for stmt in self.visit(node.start):
1045 yield stmt
1047 for stmt in self.visit(node.content):
1048 yield stmt
1050 if node.end is not None:
1051 for stmt in self.visit(node.end):
1052 yield stmt
1054 def visit_Module(self, node):
1055 body = []
1057 body += template("import re")
1058 body += template("import functools")
1059 body += template("from itertools import chain as __chain")
1060 if version < (3, 0, 0):
1061 body += template("from sys import exc_clear as __exc_clear")
1062 else:
1063 body += template("from sys import intern")
1064 body += template("__default = intern('__default__')")
1065 body += template("__marker = object()")
1066 body += template(
1067 r"g_re_amp = re.compile(r'&(?!([A-Za-z]+|#[0-9]+);)')"
1068 )
1069 body += template(
1070 r"g_re_needs_escape = re.compile(r'[&<>\"\']').search")
1072 body += template(
1073 r"__re_whitespace = "
1074 r"functools.partial(re.compile('\\s+').sub, ' ')",
1075 )
1077 # Visit module content
1078 program = self.visit(node.program)
1080 body += [ast.FunctionDef(
1081 name=node.name, args=ast.arguments(
1082 args=[param(b) for b in self._builtins],
1083 defaults=(),
1084 ),
1085 body=program
1086 )]
1088 return body
1090 def visit_MacroProgram(self, node):
1091 functions = []
1093 # Visit defined macros
1094 macros = getattr(node, "macros", ())
1095 names = []
1096 for macro in macros:
1097 stmts = self.visit(macro)
1098 function = stmts[-1]
1099 names.append(function.name)
1100 functions += stmts
1102 # Return function dictionary
1103 functions += [ast.Return(value=ast.Dict(
1104 keys=[ast.Str(s=name) for name in names],
1105 values=[load(name) for name in names],
1106 ))]
1108 return functions
1110 def visit_Context(self, node):
1111 return template("getitem = econtext.__getitem__") + \
1112 template("get = econtext.get") + \
1113 self.visit(node.node)
1115 def visit_Macro(self, node):
1116 body = []
1118 # Initialization
1119 body += template("__append = __stream.append")
1120 body += template("__re_amp = g_re_amp")
1121 body += template("__token = None")
1122 body += template("__re_needs_escape = g_re_needs_escape")
1124 body += emit_func_convert("__convert")
1125 body += emit_func_convert_and_escape("__quote")
1127 # Resolve defaults
1128 for name in self.defaults:
1129 body += template(
1130 "NAME = econtext[KEY]",
1131 NAME=name, KEY=ast.Str(s="__" + name)
1132 )
1134 # Internal set of defined slots
1135 self._slots = set()
1137 # Visit macro body
1138 nodes = itertools.chain(*tuple(map(self.visit, node.body)))
1140 # Slot resolution
1141 for name in self._slots:
1142 body += template(
1143 "try: NAME = econtext[KEY].pop()\n"
1144 "except: NAME = None",
1145 KEY=ast.Str(s=name), NAME=store(name))
1147 exc = template(
1148 "exc_info()[1]", exc_info=Symbol(sys.exc_info), mode="eval"
1149 )
1151 exc_handler = template(
1152 "if pos is not None: rcontext.setdefault('__error__', [])."
1153 "append(token + (__filename, exc, ))",
1154 exc=exc,
1155 token=template("__tokens[pos]", pos="__token", mode="eval"),
1156 pos="__token"
1157 ) + template("raise")
1159 # Wrap visited nodes in try-except error handler.
1160 body += [
1161 ast.TryExcept(
1162 body=nodes,
1163 handlers=[ast.ExceptHandler(body=exc_handler)]
1164 )
1165 ]
1167 function_name = "render" if node.name is None else \
1168 "render_%s" % mangle(node.name)
1170 function = ast.FunctionDef(
1171 name=function_name, args=ast.arguments(
1172 args=[
1173 param("__stream"),
1174 param("econtext"),
1175 param("rcontext"),
1176 param("__i18n_domain"),
1177 param("__i18n_context"),
1178 ],
1179 defaults=[load("None"), load("None")],
1180 ),
1181 body=body
1182 )
1184 yield function
1186 def visit_Text(self, node):
1187 yield EmitText(node.value)
1189 def visit_Domain(self, node):
1190 backup = "__previous_i18n_domain_%s" % mangle(id(node))
1191 return template("BACKUP = __i18n_domain", BACKUP=backup) + \
1192 template("__i18n_domain = NAME", NAME=ast.Str(s=node.name)) + \
1193 self.visit(node.node) + \
1194 template("__i18n_domain = BACKUP", BACKUP=backup)
1196 def visit_TxContext(self, node):
1197 backup = "__previous_i18n_context_%s" % mangle(id(node))
1198 return template("BACKUP = __i18n_context", BACKUP=backup) + \
1199 template("__i18n_context = NAME", NAME=ast.Str(s=node.name)) + \
1200 self.visit(node.node) + \
1201 template("__i18n_context = BACKUP", BACKUP=backup)
1203 def visit_OnError(self, node):
1204 body = []
1206 fallback = identifier("__fallback")
1207 body += template("fallback = len(__stream)", fallback=fallback)
1209 self._enter_assignment((node.name, ))
1210 fallback_body = self.visit(node.fallback)
1211 self._leave_assignment((node.name, ))
1213 error_assignment = template(
1214 "econtext[key] = cls(__exc, __tokens[__token][1:3])\n"
1215 "if handler is not None: handler(__exc)",
1216 cls=ErrorInfo,
1217 handler=load("on_error_handler"),
1218 key=ast.Str(s=node.name),
1219 )
1221 body += [ast.TryExcept(
1222 body=self.visit(node.node),
1223 handlers=[ast.ExceptHandler(
1224 type=ast.Tuple(elts=[Builtin("Exception")], ctx=ast.Load()),
1225 name=store("__exc"),
1226 body=(error_assignment + \
1227 template("del __stream[fallback:]", fallback=fallback) + \
1228 fallback_body
1229 ),
1230 )]
1231 )]
1233 return body
1235 def visit_Content(self, node):
1236 name = "__content"
1237 body = self._engine(node.expression, store(name))
1239 if node.translate:
1240 body += emit_translate(
1241 name, name, load_econtext("target_language")
1242 )
1244 if node.char_escape:
1245 body += template(
1246 "NAME=__quote(NAME, None, '\255', None, None)",
1247 NAME=name,
1248 )
1249 else:
1250 body += template("NAME = __convert(NAME)", NAME=name)
1252 body += template("if NAME is not None: __append(NAME)", NAME=name)
1254 return body
1256 def visit_Interpolation(self, node):
1257 name = identifier("content")
1258 return self._engine(node, name) + \
1259 emit_node_if_non_trivial(name)
1261 def visit_Alias(self, node):
1262 assert len(node.names) == 1
1263 name = node.names[0]
1264 target = self._aliases[-1][name] = identifier(name, id(node))
1265 return self._engine(node.expression, target)
1267 def visit_Assignment(self, node):
1268 for name in node.names:
1269 if name in COMPILER_INTERNALS_OR_DISALLOWED:
1270 raise TranslationError(
1271 "Name disallowed by compiler.", name
1272 )
1274 if name.startswith('__'):
1275 raise TranslationError(
1276 "Name disallowed by compiler (double underscore).",
1277 name
1278 )
1280 assignment = self._engine(node.expression, store("__value"))
1282 if len(node.names) != 1:
1283 target = ast.Tuple(
1284 elts=[store_econtext(name) for name in node.names],
1285 ctx=ast.Store(),
1286 )
1287 else:
1288 target = store_econtext(node.names[0])
1290 assignment.append(ast.Assign(targets=[target], value=load("__value")))
1292 for name in node.names:
1293 if not node.local:
1294 assignment += template(
1295 "rcontext[KEY] = __value", KEY=ast.Str(s=native_string(name))
1296 )
1298 return assignment
1300 def visit_Define(self, node):
1301 scope = set(self._scopes[-1])
1302 self._scopes.append(scope)
1303 self._aliases.append(self._aliases[-1].copy())
1305 for assignment in node.assignments:
1306 if assignment.local:
1307 for stmt in self._enter_assignment(assignment.names):
1308 yield stmt
1310 for stmt in self.visit(assignment):
1311 yield stmt
1313 for stmt in self.visit(node.node):
1314 yield stmt
1316 for assignment in node.assignments:
1317 if assignment.local:
1318 for stmt in self._leave_assignment(assignment.names):
1319 yield stmt
1321 self._scopes.pop()
1322 self._aliases.pop()
1324 def visit_Omit(self, node):
1325 return self.visit_Condition(node)
1327 def visit_Condition(self, node):
1328 target = "__condition"
1330 def step(expressions, body, condition):
1331 for i, expression in enumerate(reversed(expressions)):
1332 stmts = evaluate(expression, body)
1333 if i > 0:
1334 stmts.append(
1335 ast.If(
1336 ast.Compare(
1337 left=load(target),
1338 ops=[ast.Is()],
1339 comparators=[load(str(condition))]
1340 ),
1341 body,
1342 None
1343 )
1344 )
1345 body = stmts
1346 return body
1348 def evaluate(node, body=None):
1349 if isinstance(node, Logical):
1350 condition = isinstance(node, And)
1351 return step(node.expressions, body, condition)
1353 return self._engine(node, target)
1355 body = evaluate(node.expression)
1356 orelse = getattr(node, "orelse", None)
1358 body.append(
1359 ast.If(
1360 test=load(target),
1361 body=self.visit(node.node) or [ast.Pass()],
1362 orelse=self.visit(orelse) if orelse else None,
1363 )
1364 )
1366 return body
1368 def visit_Translate(self, node):
1369 """Translation.
1371 Visit items and assign output to a default value.
1373 Finally, compile a translation expression and use either
1374 result or default.
1375 """
1377 body = []
1379 # Track the blocks of this translation
1380 self._translations.append(set())
1382 # Prepare new stream
1383 append = identifier("append", id(node))
1384 stream = identifier("stream", id(node))
1385 body += template("s = new_list", s=stream, new_list=LIST) + \
1386 template("a = s.append", a=append, s=stream)
1388 # Visit body to generate the message body
1389 code = self.visit(node.node)
1390 body.append(Scope(code, append, stream))
1392 # Reduce white space and assign as message id
1393 msgid = identifier("msgid", id(node))
1394 body += template(
1395 "msgid = __re_whitespace(''.join(stream)).strip()",
1396 msgid=msgid, stream=stream
1397 )
1399 default = msgid
1401 # Compute translation block mapping if applicable
1402 names = self._translations[-1]
1403 if names:
1404 keys = []
1405 values = []
1407 for name in names:
1408 stream, append = self._get_translation_identifiers(name)
1409 keys.append(ast.Str(s=name))
1410 values.append(load(stream))
1412 # Initialize value
1413 body.insert(
1414 0, ast.Assign(
1415 targets=[store(stream)],
1416 value=ast.Str(s=native_string(""))))
1418 mapping = ast.Dict(keys=keys, values=values)
1419 else:
1420 mapping = None
1422 # if this translation node has a name, use it as the message id
1423 if node.msgid:
1424 msgid = ast.Str(s=node.msgid)
1426 # emit the translation expression
1427 body += template(
1428 "if msgid: __append(translate("
1429 "msgid, mapping=mapping, default=default, domain=__i18n_domain, context=__i18n_context, target_language=target_language))",
1430 msgid=msgid, default=default, mapping=mapping,
1431 target_language=load_econtext("target_language")
1432 )
1434 # pop away translation block reference
1435 self._translations.pop()
1437 return body
1439 def visit_Start(self, node):
1440 try:
1441 line, column = node.prefix.location
1442 except AttributeError:
1443 line, column = 0, 0
1445 yield Comment(
1446 " %s%s ... (%d:%d)\n"
1447 " --------------------------------------------------------" % (
1448 node.prefix, node.name, line, column))
1450 if node.attributes:
1451 yield EmitText(node.prefix + node.name)
1452 for stmt in self.visit(node.attributes):
1453 yield stmt
1455 yield EmitText(node.suffix)
1456 else:
1457 yield EmitText(node.prefix + node.name + node.suffix)
1459 def visit_End(self, node):
1460 yield EmitText(node.prefix + node.name + node.space + node.suffix)
1462 def visit_Attribute(self, node):
1463 attr_format = (node.space + node.name + node.eq +
1464 node.quote + "%s" + node.quote)
1466 filter_args = list(map(self._engine.cache.get, node.filters))
1468 filter_condition = template(
1469 "NAME not in CHAIN",
1470 NAME=ast.Str(s=node.name),
1471 CHAIN=ast.Call(
1472 func=load("__chain"),
1473 args=filter_args,
1474 keywords=[],
1475 starargs=None,
1476 kwargs=None,
1477 ),
1478 mode="eval"
1479 )
1481 # Static attributes are just outputted directly
1482 if isinstance(node.expression, ast.Str):
1483 s = attr_format % node.expression.s
1484 if node.filters:
1485 return template(
1486 "if C: __append(S)", C=filter_condition, S=ast.Str(s=s)
1487 )
1488 else:
1489 return [EmitText(s)]
1491 target = identifier("attr", node.name)
1492 body = self._engine(node.expression, store(target))
1494 condition = template("TARGET is not None", TARGET=target, mode="eval")
1496 if node.filters:
1497 condition = ast.BoolOp(
1498 values=[condition, filter_condition],
1499 op=ast.And(),
1500 )
1502 return body + template(
1503 "if CONDITION: __append(FORMAT % TARGET)",
1504 FORMAT=ast.Str(s=attr_format),
1505 TARGET=target,
1506 CONDITION=condition,
1507 )
1509 def visit_DictAttributes(self, node):
1510 target = identifier("attr", id(node))
1511 body = self._engine(node.expression, store(target))
1513 exclude = Static(template(
1514 "set(LIST)", LIST=ast.List(
1515 elts=[ast.Str(s=name) for name in node.exclude],
1516 ctx=ast.Load(),
1517 ), mode="eval"
1518 ))
1520 body += template(
1521 "for name, value in TARGET.items():\n "
1522 "if name not in EXCLUDE and value is not None: __append("
1523 "' ' + name + '=' + QUOTE + "
1524 "QUOTE_FUNC(value, QUOTE, QUOTE_ENTITY, None, None) + QUOTE"
1525 ")",
1526 TARGET=target,
1527 EXCLUDE=exclude,
1528 QUOTE_FUNC="__quote",
1529 QUOTE=ast.Str(s=node.quote),
1530 QUOTE_ENTITY=ast.Str(s=char2entity(node.quote or '\0')),
1531 )
1533 return body
1535 def visit_Cache(self, node):
1536 body = []
1538 for expression in node.expressions:
1539 # Skip re-evaluation
1540 if self._expression_cache.get(expression):
1541 continue
1543 name = identifier("cache", id(expression))
1544 target = store(name)
1546 body += self._engine(expression, target)
1547 self._expression_cache[expression] = target
1549 body += self.visit(node.node)
1551 return body
1553 def visit_Cancel(self, node):
1554 body = []
1556 for expression in node.expressions:
1557 assert self._expression_cache.get(expression) is not None
1558 name = identifier("cache", id(expression))
1559 target = store(name)
1560 body += self._engine(node.value, target)
1562 body += self.visit(node.node)
1564 return body
1566 def visit_UseInternalMacro(self, node):
1567 if node.name is None:
1568 render = "render"
1569 else:
1570 render = "render_%s" % mangle(node.name)
1571 token_reset = template("__token = None")
1572 return token_reset + template(
1573 "f(__stream, econtext.copy(), rcontext, __i18n_domain)",
1574 f=render) + \
1575 template("econtext.update(rcontext)")
1577 def visit_DefineSlot(self, node):
1578 name = "__slot_%s" % mangle(node.name)
1579 body = self.visit(node.node)
1581 self._slots.add(name)
1583 orelse = template(
1584 "SLOT(__stream, econtext.copy(), rcontext)",
1585 SLOT=name)
1586 test = ast.Compare(
1587 left=load(name),
1588 ops=[ast.Is()],
1589 comparators=[load("None")]
1590 )
1592 return [
1593 ast.If(test=test, body=body or [ast.Pass()], orelse=orelse)
1594 ]
1596 def visit_Name(self, node):
1597 """Translation name."""
1599 if not self._translations:
1600 raise TranslationError(
1601 "Not allowed outside of translation.", node.name)
1603 if node.name in self._translations[-1]:
1604 raise TranslationError(
1605 "Duplicate translation name: %s.", node.name)
1607 self._translations[-1].add(node.name)
1608 body = []
1610 # prepare new stream
1611 stream, append = self._get_translation_identifiers(node.name)
1612 body += template("s = new_list", s=stream, new_list=LIST) + \
1613 template("a = s.append", a=append, s=stream)
1615 # generate code
1616 code = self.visit(node.node)
1617 body.append(Scope(code, append))
1619 # output msgid
1620 text = Text('${%s}' % node.name)
1621 body += self.visit(text)
1623 # Concatenate stream
1624 body += template("stream = ''.join(stream)", stream=stream)
1626 return body
1628 def visit_CodeBlock(self, node):
1629 stmts = template(textwrap.dedent(node.source.strip('\n')))
1631 for stmt in stmts:
1632 self._visitor(stmt)
1634 return set_token(stmts, node.source)
1636 def visit_UseExternalMacro(self, node):
1637 self._macros.append(node.extend)
1639 callbacks = []
1640 for slot in node.slots:
1641 key = "__slot_%s" % mangle(slot.name)
1642 fun = "__fill_%s" % mangle(slot.name)
1644 self._current_slot.append(slot.name)
1646 body = template("getitem = econtext.__getitem__") + \
1647 template("get = econtext.get") + \
1648 self.visit(slot.node)
1650 assert self._current_slot.pop() == slot.name
1652 callbacks.append(
1653 ast.FunctionDef(
1654 name=fun,
1655 args=ast.arguments(
1656 args=[
1657 param("__stream"),
1658 param("econtext"),
1659 param("rcontext"),
1660 param("__i18n_domain"),
1661 param("__i18n_context"),
1662 ],
1663 defaults=[load("__i18n_domain"), load("__i18n_context")],
1664 ),
1665 body=body or [ast.Pass()],
1666 ))
1668 key = ast.Str(s=key)
1670 assignment = template(
1671 "_slots = econtext[KEY] = DEQUE((NAME,))",
1672 KEY=key, NAME=fun, DEQUE=Symbol(collections.deque),
1673 )
1675 if node.extend:
1676 append = template("_slots.appendleft(NAME)", NAME=fun)
1678 assignment = [ast.TryExcept(
1679 body=template("_slots = getitem(KEY)", KEY=key),
1680 handlers=[ast.ExceptHandler(body=assignment)],
1681 orelse=append,
1682 )]
1684 callbacks.extend(assignment)
1686 assert self._macros.pop() == node.extend
1688 assignment = self._engine(node.expression, store("__macro"))
1690 return (
1691 callbacks +
1692 assignment +
1693 set_token(
1694 template("__m = __macro.include"),
1695 node.expression.value
1696 ) +
1697 template(
1698 "__m(__stream, econtext.copy(), "
1699 "rcontext, __i18n_domain)"
1700 ) +
1701 template("econtext.update(rcontext)")
1702 )
1704 def visit_Repeat(self, node):
1705 # Used for loop variable definition and restore
1706 self._scopes.append(set())
1708 # Variable assignment and repeat key for single- and
1709 # multi-variable repeat clause
1710 if node.local:
1711 contexts = "econtext",
1712 else:
1713 contexts = "econtext", "rcontext"
1715 for name in node.names:
1716 if name in COMPILER_INTERNALS_OR_DISALLOWED:
1717 raise TranslationError(
1718 "Name disallowed by compiler.", name
1719 )
1721 if len(node.names) > 1:
1722 targets = [
1723 ast.Tuple(elts=[
1724 subscript(native_string(name), load(context), ast.Store())
1725 for name in node.names], ctx=ast.Store())
1726 for context in contexts
1727 ]
1729 key = ast.Tuple(
1730 elts=[ast.Str(s=name) for name in node.names],
1731 ctx=ast.Load())
1732 else:
1733 name = node.names[0]
1734 targets = [
1735 subscript(native_string(name), load(context), ast.Store())
1736 for context in contexts
1737 ]
1739 key = ast.Str(s=node.names[0])
1741 index = identifier("__index", id(node))
1742 assignment = [ast.Assign(targets=targets, value=load("__item"))]
1744 # Make repeat assignment in outer loop
1745 names = node.names
1746 local = node.local
1748 outer = self._engine(node.expression, store("__iterator"))
1750 if local:
1751 outer[:] = list(self._enter_assignment(names)) + outer
1753 outer += template(
1754 "__iterator, INDEX = getitem('repeat')(key, __iterator)",
1755 key=key, INDEX=index
1756 )
1758 # Set a trivial default value for each name assigned to make
1759 # sure we assign a value even if the iteration is empty
1760 outer += [ast.Assign(
1761 targets=[store_econtext(name)
1762 for name in node.names],
1763 value=load("None"))
1764 ]
1766 # Compute inner body
1767 inner = self.visit(node.node)
1769 # After each iteration, decrease the index
1770 inner += template("index -= 1", index=index)
1772 # For items up to N - 1, emit repeat whitespace
1773 inner += template(
1774 "if INDEX > 0: __append(WHITESPACE)",
1775 INDEX=index, WHITESPACE=ast.Str(s=node.whitespace)
1776 )
1778 # Main repeat loop
1779 outer += [ast.For(
1780 target=store("__item"),
1781 iter=load("__iterator"),
1782 body=assignment + inner,
1783 )]
1785 # Finally, clean up assignment if it's local
1786 if outer:
1787 outer += self._leave_assignment(names)
1789 self._scopes.pop()
1791 return outer
1793 def _get_translation_identifiers(self, name):
1794 assert self._translations
1795 prefix = str(id(self._translations[-1])).replace('-', '_')
1796 stream = identifier("stream_%s" % prefix, name)
1797 append = identifier("append_%s" % prefix, name)
1798 return stream, append
1800 def _enter_assignment(self, names):
1801 for name in names:
1802 for stmt in template(
1803 "BACKUP = get(KEY, __marker)",
1804 BACKUP=identifier("backup_%s" % name, id(names)),
1805 KEY=ast.Str(s=native_string(name)),
1806 ):
1807 yield stmt
1809 def _leave_assignment(self, names):
1810 for name in names:
1811 for stmt in template(
1812 "if BACKUP is __marker: del econtext[KEY]\n"
1813 "else: econtext[KEY] = BACKUP",
1814 BACKUP=identifier("backup_%s" % name, id(names)),
1815 KEY=ast.Str(s=native_string(name)),
1816 ):
1817 yield stmt