Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import re 

2import sys 

3import itertools 

4import logging 

5import threading 

6import functools 

7import collections 

8import pickle 

9import textwrap 

10 

11from .astutil import load 

12from .astutil import store 

13from .astutil import param 

14from .astutil import swap 

15from .astutil import subscript 

16from .astutil import node_annotations 

17from .astutil import annotated 

18from .astutil import NameLookupRewriteVisitor 

19from .astutil import Comment 

20from .astutil import Symbol 

21from .astutil import Builtin 

22from .astutil import Static 

23from .astutil import TokenRef 

24from .astutil import Node 

25 

26from .codegen import TemplateCodeGenerator 

27from .codegen import template 

28 

29from .tal import ErrorInfo 

30from .tal import NAME 

31from .i18n import simple_translate 

32 

33from .nodes import Text 

34from .nodes import Value 

35from .nodes import Substitution 

36from .nodes import Assignment 

37from .nodes import Module 

38from .nodes import Context 

39from .nodes import Is 

40from .nodes import IsNot 

41from .nodes import Equals 

42from .nodes import Logical 

43from .nodes import And 

44 

45from .tokenize import Token 

46from .config import DEBUG_MODE 

47from .exc import TranslationError 

48from .exc import ExpressionError 

49from .parser import groupdict 

50 

51from .utils import DebuggingOutputStream 

52from .utils import char2entity 

53from .utils import ListDictProxy 

54from .utils import native_string 

55from .utils import byte_string 

56from .utils import string_type 

57from .utils import unicode_string 

58from .utils import version 

59from .utils import ast 

60from .utils import safe_native 

61from .utils import builtins 

62from .utils import decode_htmlentities 

63from .utils import join 

64 

65if version >= (3, 0, 0): 

66 long = int 

67 

68log = logging.getLogger('chameleon.compiler') 

69 

70COMPILER_INTERNALS_OR_DISALLOWED = set([ 

71 "econtext", 

72 "rcontext", 

73 "str", 

74 "int", 

75 "float", 

76 "long", 

77 "len", 

78 "None", 

79 "True", 

80 "False", 

81 "RuntimeError", 

82 ]) 

83 

84 

85RE_MANGLE = re.compile(r'[^\w_]') 

86RE_NAME = re.compile('^%s$' % NAME) 

87 

88if DEBUG_MODE: 

89 LIST = template("cls()", cls=DebuggingOutputStream, mode="eval") 

90else: 

91 LIST = template("[]", mode="eval") 

92 

93 

94def identifier(prefix, suffix=None): 

95 return "__%s_%s" % (prefix, mangle(suffix or id(prefix))) 

96 

97 

98def mangle(string): 

99 return RE_MANGLE.sub( 

100 '_', unicode_string(string) 

101 ).replace('\n', '').replace('-', '_') 

102 

103 

104def load_econtext(name): 

105 return template("getitem(KEY)", KEY=ast.Str(s=name), mode="eval") 

106 

107 

108def store_econtext(name): 

109 name = native_string(name) 

110 return subscript(name, load("econtext"), ast.Store()) 

111 

112 

113def store_rcontext(name): 

114 name = native_string(name) 

115 return subscript(name, load("rcontext"), ast.Store()) 

116 

117 

118def set_token(stmts, token): 

119 pos = getattr(token, "pos", 0) 

120 body = template("__token = pos", pos=TokenRef(pos, len(token))) 

121 return body + stmts 

122 

123 

124def eval_token(token): 

125 try: 

126 line, column = token.location 

127 filename = token.filename 

128 except AttributeError: 

129 line, column = 0, 0 

130 filename = "<string>" 

131 

132 string = safe_native(token) 

133 

134 return template( 

135 "(string, line, col)", 

136 string=ast.Str(s=string), 

137 line=ast.Num(n=line), 

138 col=ast.Num(n=column), 

139 mode="eval" 

140 ) 

141 

142 

143emit_node_if_non_trivial = template(is_func=True, func_args=('node',), 

144 source=r""" 

145 if node is not None: 

146 __append(node) 

147""") 

148 

149 

150emit_bool = template(is_func=True, 

151 func_args=('target', 's', 'default_marker', 'default'), 

152 func_defaults=(None, None), source=r""" 

153 if target is default_marker: 

154 target = default 

155 elif target: 

156 target = s 

157 else: 

158 target = None""") 

159 

160 

161emit_convert = template(is_func=True, 

162 func_args=('target', 'encoded', 'str', 'long', 'type', 

163 'default_marker', 'default'), 

164 func_defaults=(byte_string, unicode_string, long, type, 

165 None), 

166 source=r""" 

167 if target is None: 

168 pass 

169 elif target is default_marker: 

170 target = default 

171 else: 

172 __tt = type(target) 

173 

174 if __tt is int or __tt is float or __tt is long: 

175 target = str(target) 

176 elif __tt is encoded: 

177 target = decode(target) 

178 elif __tt is not str: 

179 try: 

180 target = target.__html__ 

181 except AttributeError: 

182 __converted = convert(target) 

183 target = str(target) if target is __converted else __converted 

184 else: 

185 target = target()""") 

186 

187 

188emit_func_convert = template(is_func=True, 

189 func_args=('func', 'encoded', 'str','long','type'), 

190 func_defaults=(byte_string, unicode_string, long, 

191 type), 

192 source=r""" 

193 def func(target): 

194 if target is None: 

195 return 

196 

197 __tt = type(target) 

198 

199 if __tt is int or __tt is float or __tt is long: 

200 target = str(target) 

201 

202 elif __tt is encoded: 

203 target = decode(target) 

204 

205 elif __tt is not str: 

206 try: 

207 target = target.__html__ 

208 except AttributeError: 

209 __converted = convert(target) 

210 target = str(target) if target is __converted else __converted 

211 else: 

212 target = target() 

213 

214 return target""") 

215 

216 

217emit_translate = template(is_func=True, 

218 func_args=('target', 'msgid', 'target_language', 

219 'default'), 

220 func_defaults=(None,), 

221 source=r""" 

222 target = translate(msgid, default=default, domain=__i18n_domain, 

223 context=__i18n_context, 

224 target_language=target_language)""") 

225 

226 

227emit_func_convert_and_escape = template( 

228 is_func=True, 

229 func_args=('func', 'str', 'long', 'type', 'encoded'), 

230 func_defaults=(unicode_string, long, type, byte_string,), 

231 source=r""" 

232 def func(target, quote, quote_entity, default, default_marker): 

233 if target is None: 

234 return 

235 

236 if target is default_marker: 

237 return default 

238 

239 __tt = type(target) 

240 

241 if __tt is int or __tt is float or __tt is long: 

242 target = str(target) 

243 else: 

244 if __tt is encoded: 

245 target = decode(target) 

246 elif __tt is not str: 

247 try: 

248 target = target.__html__ 

249 except: 

250 __converted = convert(target) 

251 target = str(target) if target is __converted \ 

252 else __converted 

253 else: 

254 return target() 

255 

256 if target is not None: 

257 try: 

258 escape = __re_needs_escape(target) is not None 

259 except TypeError: 

260 pass 

261 else: 

262 if escape: 

263 # Character escape 

264 if '&' in target: 

265 target = target.replace('&', '&amp;') 

266 if '<' in target: 

267 target = target.replace('<', '&lt;') 

268 if '>' in target: 

269 target = target.replace('>', '&gt;') 

270 if quote is not None and quote in target: 

271 target = target.replace(quote, quote_entity) 

272 

273 return target""") 

274 

275 

276class EmitText(Node): 

277 """Append text to output.""" 

278 

279 _fields = "s", 

280 

281 

282class Scope(Node): 

283 """"Set a local output scope.""" 

284 

285 _fields = "body", "append", "stream" 

286 

287 body = None 

288 append = None 

289 stream = None 

290 

291 

292class Interpolator(object): 

293 braces_required_regex = re.compile( 

294 r'(\$)?\$({(?P<expression>.*)})', 

295 re.DOTALL) 

296 

297 braces_optional_regex = re.compile( 

298 r'(\$)?\$({(?P<expression>.*)}|(?P<variable>[A-Za-z][A-Za-z0-9_]*))', 

299 re.DOTALL) 

300 

301 def __init__(self, expression, braces_required, translate=False, 

302 decode_htmlentities=False): 

303 self.expression = expression 

304 self.regex = self.braces_required_regex if braces_required else \ 

305 self.braces_optional_regex 

306 self.translate = translate 

307 self.decode_htmlentities = decode_htmlentities 

308 

309 def __call__(self, name, engine): 

310 """The strategy is to find possible expression strings and 

311 call the ``validate`` function of the parser to validate. 

312 

313 For every possible starting point, the longest possible 

314 expression is tried first, then the second longest and so 

315 forth. 

316 

317 Example 1: 

318 

319 ${'expressions use the ${<expression>} format'} 

320 

321 The entire expression is attempted first and it is also the 

322 only one that validates. 

323 

324 Example 2: 

325 

326 ${'Hello'} ${'world!'} 

327 

328 Validation of the longest possible expression (the entire 

329 string) will fail, while the second round of attempts, 

330 ``${'Hello'}`` and ``${'world!'}`` respectively, validate. 

331 

332 """ 

333 

334 body = [] 

335 nodes = [] 

336 text = self.expression 

337 

338 expr_map = {} 

339 translate = self.translate 

340 

341 while text: 

342 matched = text 

343 m = self.regex.search(matched) 

344 if m is None: 

345 text = text.replace('$$', '$') 

346 nodes.append(ast.Str(s=text)) 

347 break 

348 

349 part = text[:m.start()] 

350 text = text[m.start():] 

351 

352 skip = text.startswith('$$') 

353 if skip: 

354 part = part + '$' 

355 

356 if part: 

357 part = part.replace('$$', '$') 

358 node = ast.Str(s=part) 

359 nodes.append(node) 

360 

361 if skip: 

362 text = text[2:] 

363 continue 

364 

365 if not body: 

366 target = name 

367 else: 

368 target = store("%s_%d" % (name.id, text.pos)) 

369 

370 while True: 

371 d = groupdict(m, matched) 

372 string = d["expression"] or d.get("variable") or "" 

373 

374 if self.decode_htmlentities: 

375 string = decode_htmlentities(string) 

376 

377 if string: 

378 try: 

379 compiler = engine.parse(string) 

380 body += compiler.assign_text(target) 

381 except ExpressionError: 

382 matched = matched[m.start():m.end() - 1] 

383 m = self.regex.search(matched) 

384 if m is None: 

385 raise 

386 

387 continue 

388 else: 

389 s = m.group() 

390 assign = ast.Assign(targets=[target], value=ast.Str(s=s)) 

391 body += [assign] 

392 

393 break 

394 

395 # If one or more expressions are not simple names, we 

396 # disable translation. 

397 if RE_NAME.match(string) is None: 

398 translate = False 

399 

400 # if this is the first expression, use the provided 

401 # assignment name; otherwise, generate one (here based 

402 # on the string position) 

403 node = load(target.id) 

404 nodes.append(node) 

405 

406 expr_map[node] = safe_native(string) 

407 

408 text = text[len(m.group()):] 

409 

410 if len(nodes) == 1: 

411 target = nodes[0] 

412 

413 if translate and isinstance(target, ast.Str): 

414 target = template( 

415 "translate(msgid, domain=__i18n_domain, context=__i18n_context, target_language=target_language)", 

416 msgid=target, mode="eval", 

417 target_language=load("target_language"), 

418 ) 

419 else: 

420 if translate: 

421 formatting_string = "" 

422 keys = [] 

423 values = [] 

424 

425 for node in nodes: 

426 if isinstance(node, ast.Str): 

427 formatting_string += node.s 

428 else: 

429 string = expr_map[node] 

430 formatting_string += "${%s}" % string 

431 keys.append(ast.Str(s=string)) 

432 values.append(node) 

433 

434 target = template( 

435 "translate(msgid, mapping=mapping, domain=__i18n_domain, context=__i18n_context, target_language=target_language)", 

436 msgid=ast.Str(s=formatting_string), 

437 target_language=load("target_language"), 

438 mapping=ast.Dict(keys=keys, values=values), 

439 mode="eval" 

440 ) 

441 else: 

442 nodes = [ 

443 node if isinstance(node, ast.Str) else 

444 template( 

445 "NODE if NODE is not None else ''", 

446 NODE=node, mode="eval" 

447 ) 

448 for node in nodes 

449 ] 

450 

451 target = ast.BinOp( 

452 left=ast.Str(s="%s" * len(nodes)), 

453 op=ast.Mod(), 

454 right=ast.Tuple(elts=nodes, ctx=ast.Load())) 

455 

456 body += [ast.Assign(targets=[name], value=target)] 

457 return body 

458 

459 

460class ExpressionEngine(object): 

461 """Expression engine. 

462 

463 This test demonstrates how to configure and invoke the engine. 

464 

465 >>> from chameleon import tales 

466 >>> parser = tales.ExpressionParser({ 

467 ... 'python': tales.PythonExpr, 

468 ... 'not': tales.NotExpr, 

469 ... 'exists': tales.ExistsExpr, 

470 ... 'string': tales.StringExpr, 

471 ... }, 'python') 

472 

473 >>> engine = ExpressionEngine(parser) 

474 

475 An expression evaluation function: 

476 

477 >>> eval = lambda expression: tales.test( 

478 ... tales.IdentityExpr(expression), engine) 

479 

480 We have provided 'python' as the default expression type. This 

481 means that when no prefix is given, the expression is evaluated as 

482 a Python expression: 

483 

484 >>> eval('not False') 

485 True 

486 

487 Note that the ``type`` prefixes bind left. If ``not`` and 

488 ``exits`` are two expression type prefixes, consider the 

489 following:: 

490 

491 >>> eval('not: exists: int(None)') 

492 True 

493 

494 The pipe operator binds right. In the following example, but 

495 arguments are evaluated against ``not: exists: ``. 

496 

497 >>> eval('not: exists: help') 

498 False 

499 """ 

500 

501 supported_char_escape_set = set(('&', '<', '>')) 

502 

503 def __init__(self, parser, char_escape=(), 

504 default=None, default_marker=None): 

505 self._parser = parser 

506 self._char_escape = char_escape 

507 self._default = default 

508 self._default_marker = default_marker 

509 

510 def __call__(self, string, target): 

511 # BBB: This method is deprecated. Instead, a call should first 

512 # be made to ``parse`` and then one of the assignment methods 

513 # ("value" or "text"). 

514 

515 compiler = self.parse(string) 

516 return compiler(string, target) 

517 

518 def parse(self, string, handle_errors=True, char_escape=None): 

519 expression = self._parser(string) 

520 compiler = self.get_compiler(expression, string, handle_errors, char_escape) 

521 return ExpressionCompiler(compiler, self) 

522 

523 def get_compiler(self, expression, string, handle_errors, char_escape): 

524 if char_escape is None: 

525 char_escape = self._char_escape 

526 def compiler(target, engine, result_type=None, *args): 

527 stmts = expression(target, engine) 

528 

529 if result_type is not None: 

530 method = getattr(self, '_convert_%s' % result_type) 

531 steps = method(target, char_escape, *args) 

532 stmts.extend(steps) 

533 

534 if handle_errors: 

535 return set_token(stmts, string.strip()) 

536 

537 return stmts 

538 

539 return compiler 

540 

541 def _convert_bool(self, target, char_escape, s): 

542 """Converts value given by ``target`` to a string ``s`` if the 

543 target is a true value, otherwise ``None``. 

544 """ 

545 

546 return emit_bool( 

547 target, ast.Str(s=s), 

548 default=self._default, 

549 default_marker=self._default_marker 

550 ) 

551 

552 def _convert_structure(self, target, char_escape): 

553 """Converts value given by ``target`` to structure output.""" 

554 

555 return emit_convert( 

556 target, 

557 default=self._default, 

558 default_marker=self._default_marker, 

559 ) 

560 

561 def _convert_text(self, target, char_escape): 

562 """Converts value given by ``target`` to text.""" 

563 

564 if not char_escape: 

565 return self._convert_structure(target, char_escape) 

566 

567 # This is a cop-out - we really only support a very select 

568 # set of escape characters 

569 other = set(char_escape) - self.supported_char_escape_set 

570 

571 if other: 

572 for supported in '"', '\'', '': 

573 if supported in char_escape: 

574 quote = supported 

575 break 

576 else: 

577 raise RuntimeError( 

578 "Unsupported escape set: %s." % repr(char_escape) 

579 ) 

580 else: 

581 quote = '\0' 

582 

583 entity = char2entity(quote or '\0') 

584 

585 return template( 

586 "TARGET = __quote(TARGET, QUOTE, Q_ENTITY, DEFAULT, MARKER)", 

587 TARGET=target, 

588 QUOTE=ast.Str(s=quote), 

589 Q_ENTITY=ast.Str(s=entity), 

590 DEFAULT=self._default, 

591 MARKER=self._default_marker, 

592 ) 

593 

594 

595class ExpressionCompiler(object): 

596 def __init__(self, compiler, engine): 

597 self.compiler = compiler 

598 self.engine = engine 

599 

600 def assign_bool(self, target, s): 

601 return self.compiler(target, self.engine, "bool", s) 

602 

603 def assign_text(self, target): 

604 return self.compiler(target, self.engine, "text") 

605 

606 def assign_value(self, target): 

607 return self.compiler(target, self.engine) 

608 

609 

610class ExpressionEvaluator(object): 

611 """Evaluates dynamic expression. 

612 

613 This is not particularly efficient, but supported for legacy 

614 applications. 

615 

616 >>> from chameleon import tales 

617 >>> parser = tales.ExpressionParser({'python': tales.PythonExpr}, 'python') 

618 >>> engine = functools.partial(ExpressionEngine, parser) 

619 

620 >>> evaluate = ExpressionEvaluator(engine, { 

621 ... 'foo': 'bar', 

622 ... }) 

623 

624 The evaluation function is passed the local and remote context, 

625 the expression type and finally the expression. 

626 

627 >>> evaluate({'boo': 'baz'}, {}, 'python', 'foo + boo') 

628 'barbaz' 

629 

630 The cache is now primed: 

631 

632 >>> evaluate({'boo': 'baz'}, {}, 'python', 'foo + boo') 

633 'barbaz' 

634 

635 Note that the call method supports currying of the expression 

636 argument: 

637 

638 >>> python = evaluate({'boo': 'baz'}, {}, 'python') 

639 >>> python('foo + boo') 

640 'barbaz' 

641 

642 """ 

643 

644 __slots__ = "_engine", "_cache", "_names", "_builtins" 

645 

646 def __init__(self, engine, builtins): 

647 self._engine = engine 

648 self._names, self._builtins = zip(*builtins.items()) 

649 self._cache = {} 

650 

651 def __call__(self, econtext, rcontext, expression_type, string=None): 

652 if string is None: 

653 return functools.partial( 

654 self.__call__, econtext, rcontext, expression_type 

655 ) 

656 

657 expression = "%s:%s" % (expression_type, string) 

658 

659 try: 

660 evaluate = self._cache[expression] 

661 except KeyError: 

662 assignment = Assignment(["_result"], expression, True) 

663 module = Module("evaluate", Context(assignment)) 

664 

665 compiler = Compiler( 

666 self._engine, module, "<string>", string, 

667 ('econtext', 'rcontext') + self._names 

668 ) 

669 

670 env = {} 

671 exec(compiler.code, env) 

672 evaluate = self._cache[expression] = env["evaluate"] 

673 

674 evaluate(econtext, rcontext, *self._builtins) 

675 return econtext['_result'] 

676 

677 

678class NameTransform(object): 

679 """ 

680 >>> nt = NameTransform( 

681 ... set(('foo', 'bar', )), {'boo': 'boz'}, 

682 ... ('econtext', ), 

683 ... ) 

684 

685 >>> def test(node): 

686 ... rewritten = nt(node) 

687 ... module = ast.Module([ast.fix_missing_locations(rewritten)]) 

688 ... codegen = TemplateCodeGenerator(module) 

689 ... return codegen.code 

690 

691 Any odd name: 

692 

693 >>> test(load('frobnitz')) 

694 "getitem('frobnitz')" 

695 

696 A 'builtin' name will first be looked up via ``get`` allowing fall 

697 back to the global builtin value: 

698 

699 >>> test(load('foo')) 

700 "get('foo', foo)" 

701 

702 Internal names (with two leading underscores) are left alone: 

703 

704 >>> test(load('__internal')) 

705 '__internal' 

706 

707 Compiler internals or disallowed names: 

708 

709 >>> test(load('econtext')) 

710 'econtext' 

711 

712 Aliased names: 

713 

714 >>> test(load('boo')) 

715 'boz' 

716 

717 """ 

718 

719 def __init__(self, builtins, aliases, internals): 

720 self.builtins = builtins 

721 self.aliases = aliases 

722 self.internals = internals 

723 

724 def __call__(self, node): 

725 name = node.id 

726 

727 # Don't rewrite names that begin with an underscore; they are 

728 # internal and can be assumed to be locally defined. This 

729 # policy really should be part of the template program, not 

730 # defined here in the compiler. 

731 if name.startswith('__') or name in self.internals: 

732 return node 

733 

734 if isinstance(node.ctx, ast.Store): 

735 return store_econtext(name) 

736 

737 aliased = self.aliases.get(name) 

738 if aliased is not None: 

739 return load(aliased) 

740 

741 # If the name is a Python global, first try acquiring it from 

742 # the dynamic context, then fall back to the global. 

743 if name in self.builtins: 

744 return template( 

745 "get(key, name)", 

746 mode="eval", 

747 key=ast.Str(s=name), 

748 name=load(name), 

749 ) 

750 

751 # Otherwise, simply acquire it from the dynamic context. 

752 return load_econtext(name) 

753 

754 

755class ExpressionTransform(object): 

756 """Internal wrapper to transform expression nodes into assignment 

757 statements. 

758 

759 The node input may use the provided expression engine, but other 

760 expression node types are supported such as ``Builtin`` which 

761 simply resolves a built-in name. 

762 

763 Used internally be the compiler. 

764 """ 

765 

766 loads_symbol = Symbol(pickle.loads) 

767 

768 def __init__(self, engine_factory, cache, visitor, strict=True): 

769 self.engine_factory = engine_factory 

770 self.cache = cache 

771 self.strict = strict 

772 self.visitor = visitor 

773 

774 def __call__(self, expression, target): 

775 if isinstance(target, string_type): 

776 target = store(target) 

777 

778 try: 

779 stmts = self.translate(expression, target) 

780 except ExpressionError: 

781 if self.strict: 

782 raise 

783 

784 exc = sys.exc_info()[1] 

785 p = pickle.dumps(exc, -1) 

786 

787 stmts = template( 

788 "__exc = loads(p)", loads=self.loads_symbol, p=ast.Str(s=p) 

789 ) 

790 

791 stmts += set_token([ast.Raise(exc=load("__exc"))], exc.token) 

792 

793 # Apply visitor to each statement 

794 for stmt in stmts: 

795 self.visitor(stmt) 

796 

797 return stmts 

798 

799 def translate(self, expression, target): 

800 if isinstance(target, string_type): 

801 target = store(target) 

802 

803 cached = self.cache.get(expression) 

804 

805 if cached is not None: 

806 stmts = [ast.Assign(targets=[target], value=cached)] 

807 elif isinstance(expression, ast.expr): 

808 stmts = [ast.Assign(targets=[target], value=expression)] 

809 else: 

810 # The engine interface supports simple strings, which 

811 # default to expression nodes 

812 if isinstance(expression, string_type): 

813 expression = Value(expression, True) 

814 

815 kind = type(expression).__name__ 

816 visitor = getattr(self, "visit_%s" % kind) 

817 stmts = visitor(expression, target) 

818 

819 # Add comment 

820 target_id = getattr(target, "id", target) 

821 comment = Comment(" %r -> %s" % (expression, target_id)) 

822 stmts.insert(0, comment) 

823 

824 return stmts 

825 

826 def visit_Value(self, node, target): 

827 engine = self.engine_factory( 

828 default=node.default, 

829 default_marker=node.default_marker 

830 ) 

831 compiler = engine.parse(node.value) 

832 return compiler.assign_value(target) 

833 

834 def visit_Copy(self, node, target): 

835 return self.translate(node.expression, target) 

836 

837 def visit_Substitution(self, node, target): 

838 engine = self.engine_factory( 

839 default=node.default, 

840 default_marker=node.default_marker 

841 ) 

842 compiler = engine.parse(node.value, char_escape=node.char_escape) 

843 return compiler.assign_text(target) 

844 

845 def visit_Negate(self, node, target): 

846 return self.translate(node.value, target) + \ 

847 template("TARGET = not TARGET", TARGET=target) 

848 

849 def visit_BinOp(self, node, target): 

850 expression = self.translate(node.left, "__expression") 

851 value = self.translate(node.right, "__value") 

852 

853 op = { 

854 Is: "is", 

855 IsNot: "is not", 

856 Equals: "==", 

857 }[node.op] 

858 return expression + value + \ 

859 template("TARGET = __expression %s __value" % op, TARGET=target) 

860 

861 def visit_Boolean(self, node, target): 

862 engine = self.engine_factory( 

863 default=node.default, 

864 default_marker=node.default_marker, 

865 ) 

866 compiler = engine.parse(node.value) 

867 return compiler.assign_bool(target, node.s) 

868 

869 def visit_Interpolation(self, node, target): 

870 expr = node.value 

871 if isinstance(expr, Substitution): 

872 engine = self.engine_factory( 

873 char_escape=expr.char_escape, 

874 default=expr.default, 

875 default_marker=expr.default_marker 

876 ) 

877 elif isinstance(expr, Value): 

878 engine = self.engine_factory( 

879 default=expr.default, 

880 default_marker=expr.default_marker 

881 ) 

882 else: 

883 raise RuntimeError("Bad value: %r." % node.value) 

884 

885 interpolator = Interpolator( 

886 expr.value, node.braces_required, 

887 translate=node.translation, 

888 decode_htmlentities=True 

889 ) 

890 

891 compiler = engine.get_compiler( 

892 interpolator, expr.value, True, () 

893 ) 

894 return compiler(target, engine, "text") 

895 

896 def visit_Translate(self, node, target): 

897 if node.msgid is not None: 

898 msgid = ast.Str(s=node.msgid) 

899 else: 

900 msgid = target 

901 return self.translate(node.node, target) + \ 

902 emit_translate( 

903 target, msgid, "target_language", 

904 default=target 

905 ) 

906 

907 def visit_Static(self, node, target): 

908 value = annotated(node) 

909 return [ast.Assign(targets=[target], value=value)] 

910 

911 def visit_Builtin(self, node, target): 

912 value = annotated(node) 

913 return [ast.Assign(targets=[target], value=value)] 

914 

915 def visit_Symbol(self, node, target): 

916 value = annotated(node) 

917 return template("TARGET = SYMBOL", TARGET=target, SYMBOL=node) 

918 

919 

920class Compiler(object): 

921 """Generic compiler class. 

922 

923 Iterates through nodes and yields Python statements which form a 

924 template program. 

925 """ 

926 

927 exceptions = NameError, \ 

928 ValueError, \ 

929 AttributeError, \ 

930 LookupError, \ 

931 TypeError 

932 

933 defaults = { 

934 'translate': Symbol(simple_translate), 

935 'decode': Builtin("str"), 

936 'convert': Builtin("str"), 

937 'on_error_handler': Builtin("str") 

938 } 

939 

940 lock = threading.Lock() 

941 

942 global_builtins = set(builtins.__dict__) 

943 

944 def __init__(self, engine_factory, node, filename, source, 

945 builtins={}, strict=True): 

946 self._scopes = [set()] 

947 self._expression_cache = {} 

948 self._translations = [] 

949 self._builtins = builtins 

950 self._aliases = [{}] 

951 self._macros = [] 

952 self._current_slot = [] 

953 

954 internals = COMPILER_INTERNALS_OR_DISALLOWED | \ 

955 set(self.defaults) 

956 

957 transform = NameTransform( 

958 self.global_builtins | set(builtins), 

959 ListDictProxy(self._aliases), 

960 internals, 

961 ) 

962 

963 self._visitor = visitor = NameLookupRewriteVisitor(transform) 

964 

965 self._engine = ExpressionTransform( 

966 engine_factory, 

967 self._expression_cache, 

968 visitor, 

969 strict=strict, 

970 ) 

971 

972 if isinstance(node_annotations, dict): 

973 self.lock.acquire() 

974 backup = node_annotations.copy() 

975 else: 

976 backup = None 

977 

978 try: 

979 module = ast.Module([]) 

980 module.body += self.visit(node) 

981 ast.fix_missing_locations(module) 

982 

983 class Generator(TemplateCodeGenerator): 

984 scopes = [Scope()] 

985 

986 def visit_EmitText(self, node): 

987 append = load(self.scopes[-1].append or "__append") 

988 for node in template("append(s)", append=append, s=ast.Str(s=node.s)): 

989 self.visit(node) 

990 

991 def visit_Scope(self, node): 

992 self.scopes.append(node) 

993 body = list(node.body) 

994 swap(body, load(node.append), "__append") 

995 if node.stream: 

996 swap(body, load(node.stream), "__stream") 

997 for node in body: 

998 self.visit(node) 

999 self.scopes.pop() 

1000 

1001 generator = Generator(module, source) 

1002 tokens = [ 

1003 Token(source[pos:pos + length], pos, source) 

1004 for pos, length in generator.tokens 

1005 ] 

1006 token_map_def = "__tokens = {" + ", ".join("%d: %r" % ( 

1007 token.pos, 

1008 (token, ) + token.location 

1009 ) for token in tokens) + "}" 

1010 finally: 

1011 if backup is not None: 

1012 node_annotations.clear() 

1013 node_annotations.update(backup) 

1014 self.lock.release() 

1015 

1016 self.code = "\n".join(( 

1017 "__filename = %r\n" % filename, 

1018 token_map_def, 

1019 generator.code 

1020 )) 

1021 

1022 def visit(self, node): 

1023 if node is None: 

1024 return () 

1025 kind = type(node).__name__ 

1026 visitor = getattr(self, "visit_%s" % kind) 

1027 iterator = visitor(node) 

1028 result = [] 

1029 for key, group in itertools.groupby(iterator, lambda node: node.__class__): 

1030 nodes = list(group) 

1031 if key is EmitText: 

1032 text = join(node.s for node in nodes) 

1033 nodes = [EmitText(text)] 

1034 result.extend(nodes) 

1035 return result 

1036 

1037 

1038 def visit_Sequence(self, node): 

1039 for item in node.items: 

1040 for stmt in self.visit(item): 

1041 yield stmt 

1042 

1043 def visit_Element(self, node): 

1044 for stmt in self.visit(node.start): 

1045 yield stmt 

1046 

1047 for stmt in self.visit(node.content): 

1048 yield stmt 

1049 

1050 if node.end is not None: 

1051 for stmt in self.visit(node.end): 

1052 yield stmt 

1053 

1054 def visit_Module(self, node): 

1055 body = [] 

1056 

1057 body += template("import re") 

1058 body += template("import functools") 

1059 body += template("from itertools import chain as __chain") 

1060 if version < (3, 0, 0): 

1061 body += template("from sys import exc_clear as __exc_clear") 

1062 else: 

1063 body += template("from sys import intern") 

1064 body += template("__default = intern('__default__')") 

1065 body += template("__marker = object()") 

1066 body += template( 

1067 r"g_re_amp = re.compile(r'&(?!([A-Za-z]+|#[0-9]+);)')" 

1068 ) 

1069 body += template( 

1070 r"g_re_needs_escape = re.compile(r'[&<>\"\']').search") 

1071 

1072 body += template( 

1073 r"__re_whitespace = " 

1074 r"functools.partial(re.compile('\\s+').sub, ' ')", 

1075 ) 

1076 

1077 # Visit module content 

1078 program = self.visit(node.program) 

1079 

1080 body += [ast.FunctionDef( 

1081 name=node.name, args=ast.arguments( 

1082 args=[param(b) for b in self._builtins], 

1083 defaults=(), 

1084 ), 

1085 body=program 

1086 )] 

1087 

1088 return body 

1089 

1090 def visit_MacroProgram(self, node): 

1091 functions = [] 

1092 

1093 # Visit defined macros 

1094 macros = getattr(node, "macros", ()) 

1095 names = [] 

1096 for macro in macros: 

1097 stmts = self.visit(macro) 

1098 function = stmts[-1] 

1099 names.append(function.name) 

1100 functions += stmts 

1101 

1102 # Return function dictionary 

1103 functions += [ast.Return(value=ast.Dict( 

1104 keys=[ast.Str(s=name) for name in names], 

1105 values=[load(name) for name in names], 

1106 ))] 

1107 

1108 return functions 

1109 

1110 def visit_Context(self, node): 

1111 return template("getitem = econtext.__getitem__") + \ 

1112 template("get = econtext.get") + \ 

1113 self.visit(node.node) 

1114 

1115 def visit_Macro(self, node): 

1116 body = [] 

1117 

1118 # Initialization 

1119 body += template("__append = __stream.append") 

1120 body += template("__re_amp = g_re_amp") 

1121 body += template("__token = None") 

1122 body += template("__re_needs_escape = g_re_needs_escape") 

1123 

1124 body += emit_func_convert("__convert") 

1125 body += emit_func_convert_and_escape("__quote") 

1126 

1127 # Resolve defaults 

1128 for name in self.defaults: 

1129 body += template( 

1130 "NAME = econtext[KEY]", 

1131 NAME=name, KEY=ast.Str(s="__" + name) 

1132 ) 

1133 

1134 # Internal set of defined slots 

1135 self._slots = set() 

1136 

1137 # Visit macro body 

1138 nodes = itertools.chain(*tuple(map(self.visit, node.body))) 

1139 

1140 # Slot resolution 

1141 for name in self._slots: 

1142 body += template( 

1143 "try: NAME = econtext[KEY].pop()\n" 

1144 "except: NAME = None", 

1145 KEY=ast.Str(s=name), NAME=store(name)) 

1146 

1147 exc = template( 

1148 "exc_info()[1]", exc_info=Symbol(sys.exc_info), mode="eval" 

1149 ) 

1150 

1151 exc_handler = template( 

1152 "if pos is not None: rcontext.setdefault('__error__', [])." 

1153 "append(token + (__filename, exc, ))", 

1154 exc=exc, 

1155 token=template("__tokens[pos]", pos="__token", mode="eval"), 

1156 pos="__token" 

1157 ) + template("raise") 

1158 

1159 # Wrap visited nodes in try-except error handler. 

1160 body += [ 

1161 ast.TryExcept( 

1162 body=nodes, 

1163 handlers=[ast.ExceptHandler(body=exc_handler)] 

1164 ) 

1165 ] 

1166 

1167 function_name = "render" if node.name is None else \ 

1168 "render_%s" % mangle(node.name) 

1169 

1170 function = ast.FunctionDef( 

1171 name=function_name, args=ast.arguments( 

1172 args=[ 

1173 param("__stream"), 

1174 param("econtext"), 

1175 param("rcontext"), 

1176 param("__i18n_domain"), 

1177 param("__i18n_context"), 

1178 ], 

1179 defaults=[load("None"), load("None")], 

1180 ), 

1181 body=body 

1182 ) 

1183 

1184 yield function 

1185 

1186 def visit_Text(self, node): 

1187 yield EmitText(node.value) 

1188 

1189 def visit_Domain(self, node): 

1190 backup = "__previous_i18n_domain_%s" % mangle(id(node)) 

1191 return template("BACKUP = __i18n_domain", BACKUP=backup) + \ 

1192 template("__i18n_domain = NAME", NAME=ast.Str(s=node.name)) + \ 

1193 self.visit(node.node) + \ 

1194 template("__i18n_domain = BACKUP", BACKUP=backup) 

1195 

1196 def visit_TxContext(self, node): 

1197 backup = "__previous_i18n_context_%s" % mangle(id(node)) 

1198 return template("BACKUP = __i18n_context", BACKUP=backup) + \ 

1199 template("__i18n_context = NAME", NAME=ast.Str(s=node.name)) + \ 

1200 self.visit(node.node) + \ 

1201 template("__i18n_context = BACKUP", BACKUP=backup) 

1202 

1203 def visit_OnError(self, node): 

1204 body = [] 

1205 

1206 fallback = identifier("__fallback") 

1207 body += template("fallback = len(__stream)", fallback=fallback) 

1208 

1209 self._enter_assignment((node.name, )) 

1210 fallback_body = self.visit(node.fallback) 

1211 self._leave_assignment((node.name, )) 

1212 

1213 error_assignment = template( 

1214 "econtext[key] = cls(__exc, __tokens[__token][1:3])\n" 

1215 "if handler is not None: handler(__exc)", 

1216 cls=ErrorInfo, 

1217 handler=load("on_error_handler"), 

1218 key=ast.Str(s=node.name), 

1219 ) 

1220 

1221 body += [ast.TryExcept( 

1222 body=self.visit(node.node), 

1223 handlers=[ast.ExceptHandler( 

1224 type=ast.Tuple(elts=[Builtin("Exception")], ctx=ast.Load()), 

1225 name=store("__exc"), 

1226 body=(error_assignment + \ 

1227 template("del __stream[fallback:]", fallback=fallback) + \ 

1228 fallback_body 

1229 ), 

1230 )] 

1231 )] 

1232 

1233 return body 

1234 

1235 def visit_Content(self, node): 

1236 name = "__content" 

1237 body = self._engine(node.expression, store(name)) 

1238 

1239 if node.translate: 

1240 body += emit_translate( 

1241 name, name, load_econtext("target_language") 

1242 ) 

1243 

1244 if node.char_escape: 

1245 body += template( 

1246 "NAME=__quote(NAME, None, '\255', None, None)", 

1247 NAME=name, 

1248 ) 

1249 else: 

1250 body += template("NAME = __convert(NAME)", NAME=name) 

1251 

1252 body += template("if NAME is not None: __append(NAME)", NAME=name) 

1253 

1254 return body 

1255 

1256 def visit_Interpolation(self, node): 

1257 name = identifier("content") 

1258 return self._engine(node, name) + \ 

1259 emit_node_if_non_trivial(name) 

1260 

1261 def visit_Alias(self, node): 

1262 assert len(node.names) == 1 

1263 name = node.names[0] 

1264 target = self._aliases[-1][name] = identifier(name, id(node)) 

1265 return self._engine(node.expression, target) 

1266 

1267 def visit_Assignment(self, node): 

1268 for name in node.names: 

1269 if name in COMPILER_INTERNALS_OR_DISALLOWED: 

1270 raise TranslationError( 

1271 "Name disallowed by compiler.", name 

1272 ) 

1273 

1274 if name.startswith('__'): 

1275 raise TranslationError( 

1276 "Name disallowed by compiler (double underscore).", 

1277 name 

1278 ) 

1279 

1280 assignment = self._engine(node.expression, store("__value")) 

1281 

1282 if len(node.names) != 1: 

1283 target = ast.Tuple( 

1284 elts=[store_econtext(name) for name in node.names], 

1285 ctx=ast.Store(), 

1286 ) 

1287 else: 

1288 target = store_econtext(node.names[0]) 

1289 

1290 assignment.append(ast.Assign(targets=[target], value=load("__value"))) 

1291 

1292 for name in node.names: 

1293 if not node.local: 

1294 assignment += template( 

1295 "rcontext[KEY] = __value", KEY=ast.Str(s=native_string(name)) 

1296 ) 

1297 

1298 return assignment 

1299 

1300 def visit_Define(self, node): 

1301 scope = set(self._scopes[-1]) 

1302 self._scopes.append(scope) 

1303 self._aliases.append(self._aliases[-1].copy()) 

1304 

1305 for assignment in node.assignments: 

1306 if assignment.local: 

1307 for stmt in self._enter_assignment(assignment.names): 

1308 yield stmt 

1309 

1310 for stmt in self.visit(assignment): 

1311 yield stmt 

1312 

1313 for stmt in self.visit(node.node): 

1314 yield stmt 

1315 

1316 for assignment in node.assignments: 

1317 if assignment.local: 

1318 for stmt in self._leave_assignment(assignment.names): 

1319 yield stmt 

1320 

1321 self._scopes.pop() 

1322 self._aliases.pop() 

1323 

1324 def visit_Omit(self, node): 

1325 return self.visit_Condition(node) 

1326 

1327 def visit_Condition(self, node): 

1328 target = "__condition" 

1329 

1330 def step(expressions, body, condition): 

1331 for i, expression in enumerate(reversed(expressions)): 

1332 stmts = evaluate(expression, body) 

1333 if i > 0: 

1334 stmts.append( 

1335 ast.If( 

1336 ast.Compare( 

1337 left=load(target), 

1338 ops=[ast.Is()], 

1339 comparators=[load(str(condition))] 

1340 ), 

1341 body, 

1342 None 

1343 ) 

1344 ) 

1345 body = stmts 

1346 return body 

1347 

1348 def evaluate(node, body=None): 

1349 if isinstance(node, Logical): 

1350 condition = isinstance(node, And) 

1351 return step(node.expressions, body, condition) 

1352 

1353 return self._engine(node, target) 

1354 

1355 body = evaluate(node.expression) 

1356 orelse = getattr(node, "orelse", None) 

1357 

1358 body.append( 

1359 ast.If( 

1360 test=load(target), 

1361 body=self.visit(node.node) or [ast.Pass()], 

1362 orelse=self.visit(orelse) if orelse else None, 

1363 ) 

1364 ) 

1365 

1366 return body 

1367 

1368 def visit_Translate(self, node): 

1369 """Translation. 

1370 

1371 Visit items and assign output to a default value. 

1372 

1373 Finally, compile a translation expression and use either 

1374 result or default. 

1375 """ 

1376 

1377 body = [] 

1378 

1379 # Track the blocks of this translation 

1380 self._translations.append(set()) 

1381 

1382 # Prepare new stream 

1383 append = identifier("append", id(node)) 

1384 stream = identifier("stream", id(node)) 

1385 body += template("s = new_list", s=stream, new_list=LIST) + \ 

1386 template("a = s.append", a=append, s=stream) 

1387 

1388 # Visit body to generate the message body 

1389 code = self.visit(node.node) 

1390 body.append(Scope(code, append, stream)) 

1391 

1392 # Reduce white space and assign as message id 

1393 msgid = identifier("msgid", id(node)) 

1394 body += template( 

1395 "msgid = __re_whitespace(''.join(stream)).strip()", 

1396 msgid=msgid, stream=stream 

1397 ) 

1398 

1399 default = msgid 

1400 

1401 # Compute translation block mapping if applicable 

1402 names = self._translations[-1] 

1403 if names: 

1404 keys = [] 

1405 values = [] 

1406 

1407 for name in names: 

1408 stream, append = self._get_translation_identifiers(name) 

1409 keys.append(ast.Str(s=name)) 

1410 values.append(load(stream)) 

1411 

1412 # Initialize value 

1413 body.insert( 

1414 0, ast.Assign( 

1415 targets=[store(stream)], 

1416 value=ast.Str(s=native_string("")))) 

1417 

1418 mapping = ast.Dict(keys=keys, values=values) 

1419 else: 

1420 mapping = None 

1421 

1422 # if this translation node has a name, use it as the message id 

1423 if node.msgid: 

1424 msgid = ast.Str(s=node.msgid) 

1425 

1426 # emit the translation expression 

1427 body += template( 

1428 "if msgid: __append(translate(" 

1429 "msgid, mapping=mapping, default=default, domain=__i18n_domain, context=__i18n_context, target_language=target_language))", 

1430 msgid=msgid, default=default, mapping=mapping, 

1431 target_language=load_econtext("target_language") 

1432 ) 

1433 

1434 # pop away translation block reference 

1435 self._translations.pop() 

1436 

1437 return body 

1438 

1439 def visit_Start(self, node): 

1440 try: 

1441 line, column = node.prefix.location 

1442 except AttributeError: 

1443 line, column = 0, 0 

1444 

1445 yield Comment( 

1446 " %s%s ... (%d:%d)\n" 

1447 " --------------------------------------------------------" % ( 

1448 node.prefix, node.name, line, column)) 

1449 

1450 if node.attributes: 

1451 yield EmitText(node.prefix + node.name) 

1452 for stmt in self.visit(node.attributes): 

1453 yield stmt 

1454 

1455 yield EmitText(node.suffix) 

1456 else: 

1457 yield EmitText(node.prefix + node.name + node.suffix) 

1458 

1459 def visit_End(self, node): 

1460 yield EmitText(node.prefix + node.name + node.space + node.suffix) 

1461 

1462 def visit_Attribute(self, node): 

1463 attr_format = (node.space + node.name + node.eq + 

1464 node.quote + "%s" + node.quote) 

1465 

1466 filter_args = list(map(self._engine.cache.get, node.filters)) 

1467 

1468 filter_condition = template( 

1469 "NAME not in CHAIN", 

1470 NAME=ast.Str(s=node.name), 

1471 CHAIN=ast.Call( 

1472 func=load("__chain"), 

1473 args=filter_args, 

1474 keywords=[], 

1475 starargs=None, 

1476 kwargs=None, 

1477 ), 

1478 mode="eval" 

1479 ) 

1480 

1481 # Static attributes are just outputted directly 

1482 if isinstance(node.expression, ast.Str): 

1483 s = attr_format % node.expression.s 

1484 if node.filters: 

1485 return template( 

1486 "if C: __append(S)", C=filter_condition, S=ast.Str(s=s) 

1487 ) 

1488 else: 

1489 return [EmitText(s)] 

1490 

1491 target = identifier("attr", node.name) 

1492 body = self._engine(node.expression, store(target)) 

1493 

1494 condition = template("TARGET is not None", TARGET=target, mode="eval") 

1495 

1496 if node.filters: 

1497 condition = ast.BoolOp( 

1498 values=[condition, filter_condition], 

1499 op=ast.And(), 

1500 ) 

1501 

1502 return body + template( 

1503 "if CONDITION: __append(FORMAT % TARGET)", 

1504 FORMAT=ast.Str(s=attr_format), 

1505 TARGET=target, 

1506 CONDITION=condition, 

1507 ) 

1508 

1509 def visit_DictAttributes(self, node): 

1510 target = identifier("attr", id(node)) 

1511 body = self._engine(node.expression, store(target)) 

1512 

1513 exclude = Static(template( 

1514 "set(LIST)", LIST=ast.List( 

1515 elts=[ast.Str(s=name) for name in node.exclude], 

1516 ctx=ast.Load(), 

1517 ), mode="eval" 

1518 )) 

1519 

1520 body += template( 

1521 "for name, value in TARGET.items():\n " 

1522 "if name not in EXCLUDE and value is not None: __append(" 

1523 "' ' + name + '=' + QUOTE + " 

1524 "QUOTE_FUNC(value, QUOTE, QUOTE_ENTITY, None, None) + QUOTE" 

1525 ")", 

1526 TARGET=target, 

1527 EXCLUDE=exclude, 

1528 QUOTE_FUNC="__quote", 

1529 QUOTE=ast.Str(s=node.quote), 

1530 QUOTE_ENTITY=ast.Str(s=char2entity(node.quote or '\0')), 

1531 ) 

1532 

1533 return body 

1534 

1535 def visit_Cache(self, node): 

1536 body = [] 

1537 

1538 for expression in node.expressions: 

1539 # Skip re-evaluation 

1540 if self._expression_cache.get(expression): 

1541 continue 

1542 

1543 name = identifier("cache", id(expression)) 

1544 target = store(name) 

1545 

1546 body += self._engine(expression, target) 

1547 self._expression_cache[expression] = target 

1548 

1549 body += self.visit(node.node) 

1550 

1551 return body 

1552 

1553 def visit_Cancel(self, node): 

1554 body = [] 

1555 

1556 for expression in node.expressions: 

1557 assert self._expression_cache.get(expression) is not None 

1558 name = identifier("cache", id(expression)) 

1559 target = store(name) 

1560 body += self._engine(node.value, target) 

1561 

1562 body += self.visit(node.node) 

1563 

1564 return body 

1565 

1566 def visit_UseInternalMacro(self, node): 

1567 if node.name is None: 

1568 render = "render" 

1569 else: 

1570 render = "render_%s" % mangle(node.name) 

1571 token_reset = template("__token = None") 

1572 return token_reset + template( 

1573 "f(__stream, econtext.copy(), rcontext, __i18n_domain)", 

1574 f=render) + \ 

1575 template("econtext.update(rcontext)") 

1576 

1577 def visit_DefineSlot(self, node): 

1578 name = "__slot_%s" % mangle(node.name) 

1579 body = self.visit(node.node) 

1580 

1581 self._slots.add(name) 

1582 

1583 orelse = template( 

1584 "SLOT(__stream, econtext.copy(), rcontext)", 

1585 SLOT=name) 

1586 test = ast.Compare( 

1587 left=load(name), 

1588 ops=[ast.Is()], 

1589 comparators=[load("None")] 

1590 ) 

1591 

1592 return [ 

1593 ast.If(test=test, body=body or [ast.Pass()], orelse=orelse) 

1594 ] 

1595 

1596 def visit_Name(self, node): 

1597 """Translation name.""" 

1598 

1599 if not self._translations: 

1600 raise TranslationError( 

1601 "Not allowed outside of translation.", node.name) 

1602 

1603 if node.name in self._translations[-1]: 

1604 raise TranslationError( 

1605 "Duplicate translation name: %s.", node.name) 

1606 

1607 self._translations[-1].add(node.name) 

1608 body = [] 

1609 

1610 # prepare new stream 

1611 stream, append = self._get_translation_identifiers(node.name) 

1612 body += template("s = new_list", s=stream, new_list=LIST) + \ 

1613 template("a = s.append", a=append, s=stream) 

1614 

1615 # generate code 

1616 code = self.visit(node.node) 

1617 body.append(Scope(code, append)) 

1618 

1619 # output msgid 

1620 text = Text('${%s}' % node.name) 

1621 body += self.visit(text) 

1622 

1623 # Concatenate stream 

1624 body += template("stream = ''.join(stream)", stream=stream) 

1625 

1626 return body 

1627 

1628 def visit_CodeBlock(self, node): 

1629 stmts = template(textwrap.dedent(node.source.strip('\n'))) 

1630 

1631 for stmt in stmts: 

1632 self._visitor(stmt) 

1633 

1634 return set_token(stmts, node.source) 

1635 

1636 def visit_UseExternalMacro(self, node): 

1637 self._macros.append(node.extend) 

1638 

1639 callbacks = [] 

1640 for slot in node.slots: 

1641 key = "__slot_%s" % mangle(slot.name) 

1642 fun = "__fill_%s" % mangle(slot.name) 

1643 

1644 self._current_slot.append(slot.name) 

1645 

1646 body = template("getitem = econtext.__getitem__") + \ 

1647 template("get = econtext.get") + \ 

1648 self.visit(slot.node) 

1649 

1650 assert self._current_slot.pop() == slot.name 

1651 

1652 callbacks.append( 

1653 ast.FunctionDef( 

1654 name=fun, 

1655 args=ast.arguments( 

1656 args=[ 

1657 param("__stream"), 

1658 param("econtext"), 

1659 param("rcontext"), 

1660 param("__i18n_domain"), 

1661 param("__i18n_context"), 

1662 ], 

1663 defaults=[load("__i18n_domain"), load("__i18n_context")], 

1664 ), 

1665 body=body or [ast.Pass()], 

1666 )) 

1667 

1668 key = ast.Str(s=key) 

1669 

1670 assignment = template( 

1671 "_slots = econtext[KEY] = DEQUE((NAME,))", 

1672 KEY=key, NAME=fun, DEQUE=Symbol(collections.deque), 

1673 ) 

1674 

1675 if node.extend: 

1676 append = template("_slots.appendleft(NAME)", NAME=fun) 

1677 

1678 assignment = [ast.TryExcept( 

1679 body=template("_slots = getitem(KEY)", KEY=key), 

1680 handlers=[ast.ExceptHandler(body=assignment)], 

1681 orelse=append, 

1682 )] 

1683 

1684 callbacks.extend(assignment) 

1685 

1686 assert self._macros.pop() == node.extend 

1687 

1688 assignment = self._engine(node.expression, store("__macro")) 

1689 

1690 return ( 

1691 callbacks + 

1692 assignment + 

1693 set_token( 

1694 template("__m = __macro.include"), 

1695 node.expression.value 

1696 ) + 

1697 template( 

1698 "__m(__stream, econtext.copy(), " 

1699 "rcontext, __i18n_domain)" 

1700 ) + 

1701 template("econtext.update(rcontext)") 

1702 ) 

1703 

1704 def visit_Repeat(self, node): 

1705 # Used for loop variable definition and restore 

1706 self._scopes.append(set()) 

1707 

1708 # Variable assignment and repeat key for single- and 

1709 # multi-variable repeat clause 

1710 if node.local: 

1711 contexts = "econtext", 

1712 else: 

1713 contexts = "econtext", "rcontext" 

1714 

1715 for name in node.names: 

1716 if name in COMPILER_INTERNALS_OR_DISALLOWED: 

1717 raise TranslationError( 

1718 "Name disallowed by compiler.", name 

1719 ) 

1720 

1721 if len(node.names) > 1: 

1722 targets = [ 

1723 ast.Tuple(elts=[ 

1724 subscript(native_string(name), load(context), ast.Store()) 

1725 for name in node.names], ctx=ast.Store()) 

1726 for context in contexts 

1727 ] 

1728 

1729 key = ast.Tuple( 

1730 elts=[ast.Str(s=name) for name in node.names], 

1731 ctx=ast.Load()) 

1732 else: 

1733 name = node.names[0] 

1734 targets = [ 

1735 subscript(native_string(name), load(context), ast.Store()) 

1736 for context in contexts 

1737 ] 

1738 

1739 key = ast.Str(s=node.names[0]) 

1740 

1741 index = identifier("__index", id(node)) 

1742 assignment = [ast.Assign(targets=targets, value=load("__item"))] 

1743 

1744 # Make repeat assignment in outer loop 

1745 names = node.names 

1746 local = node.local 

1747 

1748 outer = self._engine(node.expression, store("__iterator")) 

1749 

1750 if local: 

1751 outer[:] = list(self._enter_assignment(names)) + outer 

1752 

1753 outer += template( 

1754 "__iterator, INDEX = getitem('repeat')(key, __iterator)", 

1755 key=key, INDEX=index 

1756 ) 

1757 

1758 # Set a trivial default value for each name assigned to make 

1759 # sure we assign a value even if the iteration is empty 

1760 outer += [ast.Assign( 

1761 targets=[store_econtext(name) 

1762 for name in node.names], 

1763 value=load("None")) 

1764 ] 

1765 

1766 # Compute inner body 

1767 inner = self.visit(node.node) 

1768 

1769 # After each iteration, decrease the index 

1770 inner += template("index -= 1", index=index) 

1771 

1772 # For items up to N - 1, emit repeat whitespace 

1773 inner += template( 

1774 "if INDEX > 0: __append(WHITESPACE)", 

1775 INDEX=index, WHITESPACE=ast.Str(s=node.whitespace) 

1776 ) 

1777 

1778 # Main repeat loop 

1779 outer += [ast.For( 

1780 target=store("__item"), 

1781 iter=load("__iterator"), 

1782 body=assignment + inner, 

1783 )] 

1784 

1785 # Finally, clean up assignment if it's local 

1786 if outer: 

1787 outer += self._leave_assignment(names) 

1788 

1789 self._scopes.pop() 

1790 

1791 return outer 

1792 

1793 def _get_translation_identifiers(self, name): 

1794 assert self._translations 

1795 prefix = str(id(self._translations[-1])).replace('-', '_') 

1796 stream = identifier("stream_%s" % prefix, name) 

1797 append = identifier("append_%s" % prefix, name) 

1798 return stream, append 

1799 

1800 def _enter_assignment(self, names): 

1801 for name in names: 

1802 for stmt in template( 

1803 "BACKUP = get(KEY, __marker)", 

1804 BACKUP=identifier("backup_%s" % name, id(names)), 

1805 KEY=ast.Str(s=native_string(name)), 

1806 ): 

1807 yield stmt 

1808 

1809 def _leave_assignment(self, names): 

1810 for name in names: 

1811 for stmt in template( 

1812 "if BACKUP is __marker: del econtext[KEY]\n" 

1813 "else: econtext[KEY] = BACKUP", 

1814 BACKUP=identifier("backup_%s" % name, id(names)), 

1815 KEY=ast.Str(s=native_string(name)), 

1816 ): 

1817 yield stmt