Coverage for C:\leo.repo\leo-editor\leo\core\leoAtFile.py : 100%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20150323150718.1: * @file leoAtFile.py
4#@@first
5"""Classes to read and write @file nodes."""
6#@+<< imports >>
7#@+node:ekr.20041005105605.2: ** << imports >> (leoAtFile.py)
8import io
9import os
10import re
11import sys
12import tabnanny
13import time
14import tokenize
15from typing import List
16from leo.core import leoGlobals as g
17from leo.core import leoNodes
18#@-<< imports >>
19#@+others
20#@+node:ekr.20150509194251.1: ** cmd (decorator)
21def cmd(name): # pragma: no cover
22 """Command decorator for the AtFileCommands class."""
23 return g.new_cmd_decorator(name, ['c', 'atFileCommands',])
24#@+node:ekr.20160514120655.1: ** class AtFile
25class AtFile:
26 """A class implementing the atFile subcommander."""
27 #@+<< define class constants >>
28 #@+node:ekr.20131224053735.16380: *3* << define class constants >>
29 #@@nobeautify
31 # directives...
32 noDirective = 1 # not an at-directive.
33 allDirective = 2 # at-all (4.2)
34 docDirective = 3 # @doc.
35 atDirective = 4 # @<space> or @<newline>
36 codeDirective = 5 # @code
37 cDirective = 6 # @c<space> or @c<newline>
38 othersDirective = 7 # at-others
39 miscDirective = 8 # All other directives
40 startVerbatim = 9 # @verbatim Not a real directive. Used to issue warnings.
41 #@-<< define class constants >>
42 #@+others
43 #@+node:ekr.20041005105605.7: *3* at.Birth & init
44 #@+node:ekr.20041005105605.8: *4* at.ctor & helpers
45 # Note: g.getScript also call the at.__init__ and at.finishCreate().
47 def __init__(self, c):
48 """ctor for atFile class."""
49 # **Warning**: all these ivars must **also** be inited in initCommonIvars.
50 self.c = c
51 self.encoding = 'utf-8' # 2014/08/13
52 self.fileCommands = c.fileCommands
53 self.errors = 0 # Make sure at.error() works even when not inited.
54 # #2276: allow different section delims.
55 self.section_delim1 = '<<'
56 self.section_delim2 = '>>'
57 # **Only** at.writeAll manages these flags.
58 self.unchangedFiles = 0
59 # promptForDangerousWrite sets cancelFlag and yesToAll only if canCancelFlag is True.
60 self.canCancelFlag = False
61 self.cancelFlag = False
62 self.yesToAll = False
63 # User options: set in reloadSettings.
64 self.checkPythonCodeOnWrite = False
65 self.runPyFlakesOnWrite = False
66 self.underindentEscapeString = '\\-'
67 self.reloadSettings()
68 #@+node:ekr.20171113152939.1: *5* at.reloadSettings
69 def reloadSettings(self):
70 """AtFile.reloadSettings"""
71 c = self.c
72 self.checkPythonCodeOnWrite = c.config.getBool(
73 'check-python-code-on-write', default=True)
74 self.runPyFlakesOnWrite = c.config.getBool(
75 'run-pyflakes-on-write', default=False)
76 self.underindentEscapeString = c.config.getString(
77 'underindent-escape-string') or '\\-'
78 #@+node:ekr.20041005105605.10: *4* at.initCommonIvars
79 def initCommonIvars(self):
80 """
81 Init ivars common to both reading and writing.
83 The defaults set here may be changed later.
84 """
85 at = self
86 c = at.c
87 at.at_auto_encoding = c.config.default_at_auto_file_encoding
88 at.encoding = c.config.default_derived_file_encoding
89 at.endSentinelComment = ""
90 at.errors = 0
91 at.inCode = True
92 at.indent = 0 # The unit of indentation is spaces, not tabs.
93 at.language = None
94 at.output_newline = g.getOutputNewline(c=c)
95 at.page_width = None
96 at.root = None # The root (a position) of tree being read or written.
97 at.startSentinelComment = ""
98 at.startSentinelComment = ""
99 at.tab_width = c.tab_width or -4
100 at.writing_to_shadow_directory = False
101 #@+node:ekr.20041005105605.13: *4* at.initReadIvars
102 def initReadIvars(self, root, fileName):
103 at = self
104 at.initCommonIvars()
105 at.bom_encoding = None # The encoding implied by any BOM (set by g.stripBOM)
106 at.cloneSibCount = 0 # n > 1: Make sure n cloned sibs exists at next @+node sentinel
107 at.correctedLines = 0 # For perfect import.
108 at.docOut = [] # The doc part being accumulated.
109 at.done = False # True when @-leo seen.
110 at.fromString = False
111 at.importRootSeen = False
112 at.indentStack = []
113 at.lastLines = [] # The lines after @-leo
114 at.leadingWs = ""
115 at.lineNumber = 0 # New in Leo 4.4.8.
116 at.out = None
117 at.outStack = []
118 at.read_i = 0
119 at.read_lines = []
120 at.readVersion = '' # "5" for new-style thin files.
121 at.readVersion5 = False # Synonym for at.readVersion >= '5'
122 at.root = root
123 at.rootSeen = False
124 at.targetFileName = fileName # For at.writeError only.
125 at.v = None
126 at.vStack = [] # Stack of at.v values.
127 at.thinChildIndexStack = [] # number of siblings at this level.
128 at.thinNodeStack = [] # Entries are vnodes.
129 at.updateWarningGiven = False
130 #@+node:ekr.20041005105605.15: *4* at.initWriteIvars
131 def initWriteIvars(self, root):
132 """
133 Compute default values of all write-related ivars.
134 Return the finalized name of the output file.
135 """
136 at, c = self, self.c
137 if not c and c.config:
138 return None # pragma: no cover
139 make_dirs = c.config.create_nonexistent_directories
140 assert root
141 self.initCommonIvars()
142 assert at.checkPythonCodeOnWrite is not None
143 assert at.underindentEscapeString is not None
144 #
145 # Copy args
146 at.root = root
147 at.sentinels = True
148 #
149 # Override initCommonIvars.
150 if g.unitTesting:
151 at.output_newline = '\n'
152 #
153 # Set other ivars.
154 at.force_newlines_in_at_nosent_bodies = c.config.getBool(
155 'force-newlines-in-at-nosent-bodies')
156 # For at.putBody only.
157 at.outputList = []
158 # For stream output.
159 at.scanAllDirectives(root)
160 # Sets the following ivars:
161 # at.encoding
162 # at.explicitLineEnding
163 # at.language
164 # at.output_newline
165 # at.page_width
166 # at.tab_width
167 #
168 # Overrides of at.scanAllDirectives...
169 if at.language == 'python':
170 # Encoding directive overrides everything else.
171 encoding = g.getPythonEncodingFromString(root.b)
172 if encoding:
173 at.encoding = encoding
174 #
175 # Clean root.v.
176 if not at.errors and at.root:
177 at.root.v._p_changed = True
178 #
179 # #1907: Compute the file name and create directories as needed.
180 targetFileName = g.os_path_realpath(g.fullPath(c, root))
181 at.targetFileName = targetFileName # For at.writeError only.
182 #
183 # targetFileName can be empty for unit tests & @command nodes.
184 if not targetFileName: # pragma: no cover
185 targetFileName = root.h if g.unitTesting else None
186 at.targetFileName = targetFileName # For at.writeError only.
187 return targetFileName
188 #
189 # #2276: scan for section delims
190 at.scanRootForSectionDelims(root)
191 #
192 # Do nothing more if the file already exists.
193 if os.path.exists(targetFileName):
194 return targetFileName
195 #
196 # Create directories if enabled.
197 root_dir = g.os_path_dirname(targetFileName)
198 if make_dirs and root_dir: # pragma: no cover
199 ok = g.makeAllNonExistentDirectories(root_dir)
200 if not ok:
201 g.error(f"Error creating directories: {root_dir}")
202 return None
203 #
204 # Return the target file name, regardless of future problems.
205 return targetFileName
206 #@+node:ekr.20041005105605.17: *3* at.Reading
207 #@+node:ekr.20041005105605.18: *4* at.Reading (top level)
208 #@+node:ekr.20070919133659: *5* at.checkExternalFile
209 @cmd('check-external-file')
210 def checkExternalFile(self, event=None): # pragma: no cover
211 """Make sure an external file written by Leo may be read properly."""
212 c, p = self.c, self.c.p
213 if not p.isAtFileNode() and not p.isAtThinFileNode():
214 g.red('Please select an @thin or @file node')
215 return
216 fn = g.fullPath(c, p) # #1910.
217 if not g.os_path_exists(fn):
218 g.red(f"file not found: {fn}")
219 return
220 s, e = g.readFileIntoString(fn)
221 if s is None:
222 g.red(f"empty file: {fn}")
223 return
224 #
225 # Create a dummy, unconnected, VNode as the root.
226 root_v = leoNodes.VNode(context=c)
227 root = leoNodes.Position(root_v)
228 FastAtRead(c, gnx2vnode={}).read_into_root(s, fn, root)
229 #@+node:ekr.20041005105605.19: *5* at.openFileForReading & helper
230 def openFileForReading(self, fromString=False):
231 """
232 Open the file given by at.root.
233 This will be the private file for @shadow nodes.
234 """
235 at, c = self, self.c
236 is_at_shadow = self.root.isAtShadowFileNode()
237 if fromString: # pragma: no cover
238 if is_at_shadow: # pragma: no cover
239 return at.error(
240 'can not call at.read from string for @shadow files')
241 at.initReadLine(fromString)
242 return None, None
243 #
244 # Not from a string. Carefully read the file.
245 # Returns full path, including file name.
246 fn = g.fullPath(c, at.root)
247 # Remember the full path to this node.
248 at.setPathUa(at.root, fn)
249 if is_at_shadow: # pragma: no cover
250 fn = at.openAtShadowFileForReading(fn)
251 if not fn:
252 return None, None
253 assert fn
254 try:
255 # Sets at.encoding, regularizes whitespace and calls at.initReadLines.
256 s = at.readFileToUnicode(fn)
257 # #1466.
258 if s is None: # pragma: no cover
259 # The error has been given.
260 at._file_bytes = g.toEncodedString('')
261 return None, None
262 at.warnOnReadOnlyFile(fn)
263 except Exception: # pragma: no cover
264 at.error(f"unexpected exception opening: '@file {fn}'")
265 at._file_bytes = g.toEncodedString('')
266 fn, s = None, None
267 return fn, s
268 #@+node:ekr.20150204165040.4: *6* at.openAtShadowFileForReading
269 def openAtShadowFileForReading(self, fn): # pragma: no cover
270 """Open an @shadow for reading and return shadow_fn."""
271 at = self
272 x = at.c.shadowController
273 # readOneAtShadowNode should already have checked these.
274 shadow_fn = x.shadowPathName(fn)
275 shadow_exists = (g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn))
276 if not shadow_exists:
277 g.trace('can not happen: no private file',
278 shadow_fn, g.callers())
279 at.error(f"can not happen: private file does not exist: {shadow_fn}")
280 return None
281 # This method is the gateway to the shadow algorithm.
282 x.updatePublicAndPrivateFiles(at.root, fn, shadow_fn)
283 return shadow_fn
284 #@+node:ekr.20041005105605.21: *5* at.read & helpers
285 def read(self, root, fromString=None):
286 """Read an @thin or @file tree."""
287 at, c = self, self.c
288 fileName = g.fullPath(c, root) # #1341. #1889.
289 if not fileName: # pragma: no cover
290 at.error("Missing file name. Restoring @file tree from .leo file.")
291 return False
292 # Fix bug 760531: always mark the root as read, even if there was an error.
293 # Fix bug 889175: Remember the full fileName.
294 at.rememberReadPath(g.fullPath(c, root), root)
295 at.initReadIvars(root, fileName)
296 at.fromString = fromString
297 if at.errors:
298 return False # pragma: no cover
299 fileName, file_s = at.openFileForReading(fromString=fromString)
300 # #1798:
301 if file_s is None:
302 return False # pragma: no cover
303 #
304 # Set the time stamp.
305 if fileName:
306 c.setFileTimeStamp(fileName)
307 elif not fileName and not fromString and not file_s: # pragma: no cover
308 return False
309 root.clearVisitedInTree()
310 at.scanAllDirectives(root)
311 # Sets the following ivars:
312 # at.encoding: **changed later** by readOpenFile/at.scanHeader.
313 # at.explicitLineEnding
314 # at.language
315 # at.output_newline
316 # at.page_width
317 # at.tab_width
318 gnx2vnode = c.fileCommands.gnxDict
319 contents = fromString or file_s
320 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
321 root.clearDirty()
322 return True
323 #@+node:ekr.20071105164407: *6* at.deleteUnvisitedNodes
324 def deleteUnvisitedNodes(self, root): # pragma: no cover
325 """
326 Delete unvisited nodes in root's subtree, not including root.
328 Before Leo 5.6: Move unvisited node to be children of the 'Resurrected
329 Nodes'.
330 """
331 at, c = self, self.c
332 # Find the unvisited nodes.
333 aList = [z for z in root.subtree() if not z.isVisited()]
334 if aList:
335 at.c.deletePositionsInList(aList)
336 c.redraw()
338 #@+node:ekr.20041005105605.26: *5* at.readAll & helpers
339 def readAll(self, root):
340 """Scan positions, looking for @<file> nodes to read."""
341 at, c = self, self.c
342 old_changed = c.changed
343 t1 = time.time()
344 c.init_error_dialogs()
345 files = at.findFilesToRead(root, all=True)
346 for p in files:
347 at.readFileAtPosition(p)
348 for p in files:
349 p.v.clearDirty()
350 if not g.unitTesting and files: # pragma: no cover
351 t2 = time.time()
352 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
353 c.changed = old_changed
354 c.raise_error_dialogs()
355 #@+node:ekr.20190108054317.1: *6* at.findFilesToRead
356 def findFilesToRead(self, root, all): # pragma: no cover
358 c = self.c
359 p = root.copy()
360 scanned_nodes = set()
361 files = []
362 after = None if all else p.nodeAfterTree()
363 while p and p != after:
364 data = (p.gnx, g.fullPath(c, p))
365 # skip clones referring to exactly the same paths.
366 if data in scanned_nodes:
367 p.moveToNodeAfterTree()
368 continue
369 scanned_nodes.add(data)
370 if not p.h.startswith('@'):
371 p.moveToThreadNext()
372 elif p.isAtIgnoreNode():
373 if p.isAnyAtFileNode():
374 c.ignored_at_file_nodes.append(p.h)
375 p.moveToNodeAfterTree()
376 elif (
377 p.isAtThinFileNode() or
378 p.isAtAutoNode() or
379 p.isAtEditNode() or
380 p.isAtShadowFileNode() or
381 p.isAtFileNode() or
382 p.isAtCleanNode() # 1134.
383 ):
384 files.append(p.copy())
385 p.moveToNodeAfterTree()
386 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
387 # Note (see #1081): @asis and @nosent can *not* be updated automatically.
388 # Doing so using refresh-from-disk will delete all child nodes.
389 p.moveToNodeAfterTree()
390 else:
391 p.moveToThreadNext()
392 return files
393 #@+node:ekr.20190108054803.1: *6* at.readFileAtPosition
394 def readFileAtPosition(self, p): # pragma: no cover
395 """Read the @<file> node at p."""
396 at, c, fileName = self, self.c, p.anyAtFileNodeName()
397 if p.isAtThinFileNode() or p.isAtFileNode():
398 at.read(p)
399 elif p.isAtAutoNode():
400 at.readOneAtAutoNode(p)
401 elif p.isAtEditNode():
402 at.readOneAtEditNode(fileName, p)
403 elif p.isAtShadowFileNode():
404 at.readOneAtShadowNode(fileName, p)
405 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
406 at.rememberReadPath(g.fullPath(c, p), p)
407 elif p.isAtCleanNode():
408 at.readOneAtCleanNode(p)
409 #@+node:ekr.20220121052056.1: *5* at.readAllSelected
410 def readAllSelected(self, root): # pragma: no cover
411 """Read all @<file> nodes in root's tree."""
412 at, c = self, self.c
413 old_changed = c.changed
414 t1 = time.time()
415 c.init_error_dialogs()
416 files = at.findFilesToRead(root, all=False)
417 for p in files:
418 at.readFileAtPosition(p)
419 for p in files:
420 p.v.clearDirty()
421 if not g.unitTesting: # pragma: no cover
422 if files:
423 t2 = time.time()
424 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
425 else:
426 g.es("no @<file> nodes in the selected tree")
427 c.changed = old_changed
428 c.raise_error_dialogs()
429 #@+node:ekr.20080801071227.7: *5* at.readAtShadowNodes
430 def readAtShadowNodes(self, p): # pragma: no cover
431 """Read all @shadow nodes in the p's tree."""
432 at = self
433 after = p.nodeAfterTree()
434 p = p.copy() # Don't change p in the caller.
435 while p and p != after: # Don't use iterator.
436 if p.isAtShadowFileNode():
437 fileName = p.atShadowFileNodeName()
438 at.readOneAtShadowNode(fileName, p)
439 p.moveToNodeAfterTree()
440 else:
441 p.moveToThreadNext()
442 #@+node:ekr.20070909100252: *5* at.readOneAtAutoNode
443 def readOneAtAutoNode(self, p): # pragma: no cover
444 """Read an @auto file into p. Return the *new* position."""
445 at, c, ic = self, self.c, self.c.importCommands
446 fileName = g.fullPath(c, p) # #1521, #1341, #1914.
447 if not g.os_path_exists(fileName):
448 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
449 return p
450 # Remember that we have seen the @auto node.
451 # Fix bug 889175: Remember the full fileName.
452 at.rememberReadPath(fileName, p)
453 # if not g.unitTesting: g.es("reading:", p.h)
454 try:
455 # For #451: return p.
456 old_p = p.copy()
457 at.scanAllDirectives(p)
458 p.v.b = '' # Required for @auto API checks.
459 p.v._deleteAllChildren()
460 p = ic.createOutline(parent=p.copy())
461 # Do *not* select a position here.
462 # That would improperly expand nodes.
463 # c.selectPosition(p)
464 except Exception:
465 p = old_p
466 ic.errors += 1
467 g.es_print('Unexpected exception importing', fileName)
468 g.es_exception()
469 if ic.errors:
470 g.error(f"errors inhibited read @auto {fileName}")
471 elif c.persistenceController:
472 c.persistenceController.update_after_read_foreign_file(p)
473 # Finish.
474 if ic.errors or not g.os_path_exists(fileName):
475 p.clearDirty()
476 else:
477 g.doHook('after-auto', c=c, p=p)
478 return p
479 #@+node:ekr.20090225080846.3: *5* at.readOneAtEditNode
480 def readOneAtEditNode(self, fn, p): # pragma: no cover
481 at = self
482 c = at.c
483 ic = c.importCommands
484 # #1521
485 fn = g.fullPath(c, p)
486 junk, ext = g.os_path_splitext(fn)
487 # Fix bug 889175: Remember the full fileName.
488 at.rememberReadPath(fn, p)
489 # if not g.unitTesting: g.es("reading: @edit %s" % (g.shortFileName(fn)))
490 s, e = g.readFileIntoString(fn, kind='@edit')
491 if s is None:
492 return
493 encoding = 'utf-8' if e is None else e
494 # Delete all children.
495 while p.hasChildren():
496 p.firstChild().doDelete()
497 head = ''
498 ext = ext.lower()
499 if ext in ('.html', '.htm'):
500 head = '@language html\n'
501 elif ext in ('.txt', '.text'):
502 head = '@nocolor\n'
503 else:
504 language = ic.languageForExtension(ext)
505 if language and language != 'unknown_language':
506 head = f"@language {language}\n"
507 else:
508 head = '@nocolor\n'
509 p.b = head + g.toUnicode(s, encoding=encoding, reportErrors=True)
510 g.doHook('after-edit', p=p)
511 #@+node:ekr.20190201104956.1: *5* at.readOneAtAsisNode
512 def readOneAtAsisNode(self, fn, p): # pragma: no cover
513 """Read one @asis node. Used only by refresh-from-disk"""
514 at, c = self, self.c
515 # #1521 & #1341.
516 fn = g.fullPath(c, p)
517 junk, ext = g.os_path_splitext(fn)
518 # Remember the full fileName.
519 at.rememberReadPath(fn, p)
520 # if not g.unitTesting: g.es("reading: @asis %s" % (g.shortFileName(fn)))
521 s, e = g.readFileIntoString(fn, kind='@edit')
522 if s is None:
523 return
524 encoding = 'utf-8' if e is None else e
525 # Delete all children.
526 while p.hasChildren():
527 p.firstChild().doDelete()
528 old_body = p.b
529 p.b = g.toUnicode(s, encoding=encoding, reportErrors=True)
530 if not c.isChanged() and p.b != old_body:
531 c.setChanged()
532 #@+node:ekr.20150204165040.5: *5* at.readOneAtCleanNode & helpers
533 def readOneAtCleanNode(self, root): # pragma: no cover
534 """Update the @clean/@nosent node at root."""
535 at, c, x = self, self.c, self.c.shadowController
536 fileName = g.fullPath(c, root)
537 if not g.os_path_exists(fileName):
538 g.es_print(f"not found: {fileName}", color='red', nodeLink=root.get_UNL())
539 return False
540 at.rememberReadPath(fileName, root)
541 at.initReadIvars(root, fileName)
542 # Must be called before at.scanAllDirectives.
543 at.scanAllDirectives(root)
544 # Sets at.startSentinelComment/endSentinelComment.
545 new_public_lines = at.read_at_clean_lines(fileName)
546 old_private_lines = self.write_at_clean_sentinels(root)
547 marker = x.markerFromFileLines(old_private_lines, fileName)
548 old_public_lines, junk = x.separate_sentinels(old_private_lines, marker)
549 if old_public_lines:
550 new_private_lines = x.propagate_changed_lines(
551 new_public_lines, old_private_lines, marker, p=root)
552 else:
553 new_private_lines = []
554 root.b = ''.join(new_public_lines)
555 return True
556 if new_private_lines == old_private_lines:
557 return True
558 if not g.unitTesting:
559 g.es("updating:", root.h)
560 root.clearVisitedInTree()
561 gnx2vnode = at.fileCommands.gnxDict
562 contents = ''.join(new_private_lines)
563 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
564 return True # Errors not detected.
565 #@+node:ekr.20150204165040.7: *6* at.dump_lines
566 def dump(self, lines, tag): # pragma: no cover
567 """Dump all lines."""
568 print(f"***** {tag} lines...\n")
569 for s in lines:
570 print(s.rstrip())
571 #@+node:ekr.20150204165040.8: *6* at.read_at_clean_lines
572 def read_at_clean_lines(self, fn): # pragma: no cover
573 """Return all lines of the @clean/@nosent file at fn."""
574 at = self
575 # Use the standard helper. Better error reporting.
576 # Important: uses 'rb' to open the file.
577 s = at.openFileHelper(fn)
578 # #1798.
579 if s is None:
580 s = ''
581 else:
582 s = g.toUnicode(s, encoding=at.encoding)
583 s = s.replace('\r\n', '\n') # Suppress meaningless "node changed" messages.
584 return g.splitLines(s)
585 #@+node:ekr.20150204165040.9: *6* at.write_at_clean_sentinels
586 def write_at_clean_sentinels(self, root): # pragma: no cover
587 """
588 Return all lines of the @clean tree as if it were
589 written as an @file node.
590 """
591 at = self
592 result = at.atFileToString(root, sentinels=True)
593 s = g.toUnicode(result, encoding=at.encoding)
594 return g.splitLines(s)
595 #@+node:ekr.20080711093251.7: *5* at.readOneAtShadowNode & helper
596 def readOneAtShadowNode(self, fn, p): # pragma: no cover
598 at, c = self, self.c
599 x = c.shadowController
600 if not fn == p.atShadowFileNodeName():
601 at.error(
602 f"can not happen: fn: {fn} != atShadowNodeName: "
603 f"{p.atShadowFileNodeName()}")
604 return
605 fn = g.fullPath(c, p) # #1521 & #1341.
606 # #889175: Remember the full fileName.
607 at.rememberReadPath(fn, p)
608 shadow_fn = x.shadowPathName(fn)
609 shadow_exists = g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn)
610 # Delete all children.
611 while p.hasChildren():
612 p.firstChild().doDelete()
613 if shadow_exists:
614 at.read(p)
615 else:
616 ok = at.importAtShadowNode(p)
617 if ok:
618 # Create the private file automatically.
619 at.writeOneAtShadowNode(p)
620 #@+node:ekr.20080712080505.1: *6* at.importAtShadowNode
621 def importAtShadowNode(self, p): # pragma: no cover
622 c, ic = self.c, self.c.importCommands
623 fn = g.fullPath(c, p) # #1521, #1341, #1914.
624 if not g.os_path_exists(fn):
625 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
626 return p
627 # Delete all the child nodes.
628 while p.hasChildren():
629 p.firstChild().doDelete()
630 # Import the outline, exactly as @auto does.
631 ic.createOutline(parent=p.copy())
632 if ic.errors:
633 g.error('errors inhibited read @shadow', fn)
634 if ic.errors or not g.os_path_exists(fn):
635 p.clearDirty()
636 return ic.errors == 0
637 #@+node:ekr.20180622110112.1: *4* at.fast_read_into_root
638 def fast_read_into_root(self, c, contents, gnx2vnode, path, root): # pragma: no cover
639 """A convenience wrapper for FastAtRead.read_into_root()"""
640 return FastAtRead(c, gnx2vnode).read_into_root(contents, path, root)
641 #@+node:ekr.20041005105605.116: *4* at.Reading utils...
642 #@+node:ekr.20041005105605.119: *5* at.createImportedNode
643 def createImportedNode(self, root, headline): # pragma: no cover
644 at = self
645 if at.importRootSeen:
646 p = root.insertAsLastChild()
647 p.initHeadString(headline)
648 else:
649 # Put the text into the already-existing root node.
650 p = root
651 at.importRootSeen = True
652 p.v.setVisited() # Suppress warning about unvisited node.
653 return p
654 #@+node:ekr.20130911110233.11286: *5* at.initReadLine
655 def initReadLine(self, s):
656 """Init the ivars so that at.readLine will read all of s."""
657 at = self
658 at.read_i = 0
659 at.read_lines = g.splitLines(s)
660 at._file_bytes = g.toEncodedString(s)
661 #@+node:ekr.20041005105605.120: *5* at.parseLeoSentinel
662 def parseLeoSentinel(self, s):
663 """
664 Parse the sentinel line s.
665 If the sentinel is valid, set at.encoding, at.readVersion, at.readVersion5.
666 """
667 at, c = self, self.c
668 # Set defaults.
669 encoding = c.config.default_derived_file_encoding
670 readVersion, readVersion5 = None, None
671 new_df, start, end, isThin = False, '', '', False
672 # Example: \*@+leo-ver=5-thin-encoding=utf-8,.*/
673 pattern = re.compile(
674 r'(.+)@\+leo(-ver=([0123456789]+))?(-thin)?(-encoding=(.*)(\.))?(.*)')
675 # The old code weirdly allowed '.' in version numbers.
676 # group 1: opening delim
677 # group 2: -ver=
678 # group 3: version number
679 # group(4): -thin
680 # group(5): -encoding=utf-8,.
681 # group(6): utf-8,
682 # group(7): .
683 # group(8): closing delim.
684 m = pattern.match(s)
685 valid = bool(m)
686 if valid:
687 start = m.group(1) # start delim
688 valid = bool(start)
689 if valid:
690 new_df = bool(m.group(2)) # -ver=
691 if new_df:
692 # Set the version number.
693 if m.group(3):
694 readVersion = m.group(3)
695 readVersion5 = readVersion >= '5'
696 else:
697 valid = False # pragma: no cover
698 if valid:
699 # set isThin
700 isThin = bool(m.group(4))
701 if valid and m.group(5):
702 # set encoding.
703 encoding = m.group(6)
704 if encoding and encoding.endswith(','):
705 # Leo 4.2 or after.
706 encoding = encoding[:-1]
707 if not g.isValidEncoding(encoding): # pragma: no cover
708 g.es_print("bad encoding in derived file:", encoding)
709 valid = False
710 if valid:
711 end = m.group(8) # closing delim
712 if valid:
713 at.encoding = encoding
714 at.readVersion = readVersion
715 at.readVersion5 = readVersion5
716 return valid, new_df, start, end, isThin
717 #@+node:ekr.20130911110233.11284: *5* at.readFileToUnicode & helpers
718 def readFileToUnicode(self, fileName): # pragma: no cover
719 """
720 Carefully sets at.encoding, then uses at.encoding to convert the file
721 to a unicode string.
723 Sets at.encoding as follows:
724 1. Use the BOM, if present. This unambiguously determines the encoding.
725 2. Use the -encoding= field in the @+leo header, if present and valid.
726 3. Otherwise, uses existing value of at.encoding, which comes from:
727 A. An @encoding directive, found by at.scanAllDirectives.
728 B. The value of c.config.default_derived_file_encoding.
730 Returns the string, or None on failure.
731 """
732 at = self
733 s = at.openFileHelper(fileName) # Catches all exceptions.
734 # #1798.
735 if s is None:
736 return None
737 e, s = g.stripBOM(s)
738 if e:
739 # The BOM determines the encoding unambiguously.
740 s = g.toUnicode(s, encoding=e)
741 else:
742 # Get the encoding from the header, or the default encoding.
743 s_temp = g.toUnicode(s, 'ascii', reportErrors=False)
744 e = at.getEncodingFromHeader(fileName, s_temp)
745 s = g.toUnicode(s, encoding=e)
746 s = s.replace('\r\n', '\n')
747 at.encoding = e
748 at.initReadLine(s)
749 return s
750 #@+node:ekr.20130911110233.11285: *6* at.openFileHelper
751 def openFileHelper(self, fileName):
752 """Open a file, reporting all exceptions."""
753 at = self
754 # #1798: return None as a flag on any error.
755 s = None
756 try:
757 with open(fileName, 'rb') as f:
758 s = f.read()
759 except IOError: # pragma: no cover
760 at.error(f"can not open {fileName}")
761 except Exception: # pragma: no cover
762 at.error(f"Exception reading {fileName}")
763 g.es_exception()
764 return s
765 #@+node:ekr.20130911110233.11287: *6* at.getEncodingFromHeader
766 def getEncodingFromHeader(self, fileName, s):
767 """
768 Return the encoding given in the @+leo sentinel, if the sentinel is
769 present, or the previous value of at.encoding otherwise.
770 """
771 at = self
772 if at.errors: # pragma: no cover
773 g.trace('can not happen: at.errors > 0', g.callers())
774 e = at.encoding
775 if g.unitTesting:
776 assert False, g.callers()
777 else:
778 at.initReadLine(s)
779 old_encoding = at.encoding
780 assert old_encoding
781 at.encoding = None
782 # Execute scanHeader merely to set at.encoding.
783 at.scanHeader(fileName, giveErrors=False)
784 e = at.encoding or old_encoding
785 assert e
786 return e
787 #@+node:ekr.20041005105605.128: *5* at.readLine
788 def readLine(self):
789 """
790 Read one line from file using the present encoding.
791 Returns at.read_lines[at.read_i++]
792 """
793 # This is an old interface, now used only by at.scanHeader.
794 # For now, it's not worth replacing.
795 at = self
796 if at.read_i < len(at.read_lines):
797 s = at.read_lines[at.read_i]
798 at.read_i += 1
799 return s
800 # Not an error.
801 return '' # pragma: no cover
802 #@+node:ekr.20041005105605.129: *5* at.scanHeader
803 def scanHeader(self, fileName, giveErrors=True):
804 """
805 Scan the @+leo sentinel, using the old readLine interface.
807 Sets self.encoding, and self.start/endSentinelComment.
809 Returns (firstLines,new_df,isThinDerivedFile) where:
810 firstLines contains all @first lines,
811 new_df is True if we are reading a new-format derived file.
812 isThinDerivedFile is True if the file is an @thin file.
813 """
814 at = self
815 new_df, isThinDerivedFile = False, False
816 firstLines: List[str] = [] # The lines before @+leo.
817 s = self.scanFirstLines(firstLines)
818 valid = len(s) > 0
819 if valid:
820 valid, new_df, start, end, isThinDerivedFile = at.parseLeoSentinel(s)
821 if valid:
822 at.startSentinelComment = start
823 at.endSentinelComment = end
824 elif giveErrors: # pragma: no cover
825 at.error(f"No @+leo sentinel in: {fileName}")
826 g.trace(g.callers())
827 return firstLines, new_df, isThinDerivedFile
828 #@+node:ekr.20041005105605.130: *6* at.scanFirstLines
829 def scanFirstLines(self, firstLines): # pragma: no cover
830 """
831 Append all lines before the @+leo line to firstLines.
833 Empty lines are ignored because empty @first directives are
834 ignored.
836 We can not call sentinelKind here because that depends on the comment
837 delimiters we set here.
838 """
839 at = self
840 s = at.readLine()
841 while s and s.find("@+leo") == -1:
842 firstLines.append(s)
843 s = at.readLine()
844 return s
845 #@+node:ekr.20050103163224: *5* at.scanHeaderForThin (import code)
846 def scanHeaderForThin(self, fileName): # pragma: no cover
847 """
848 Return true if the derived file is a thin file.
850 This is a kludgy method used only by the import code."""
851 at = self
852 # Set at.encoding, regularize whitespace and call at.initReadLines.
853 at.readFileToUnicode(fileName)
854 # scanHeader uses at.readline instead of its args.
855 # scanHeader also sets at.encoding.
856 junk, junk, isThin = at.scanHeader(None)
857 return isThin
858 #@+node:ekr.20041005105605.132: *3* at.Writing
859 #@+node:ekr.20041005105605.133: *4* Writing (top level)
860 #@+node:ekr.20190111153551.1: *5* at.commands
861 #@+node:ekr.20070806105859: *6* at.writeAtAutoNodes
862 @cmd('write-at-auto-nodes')
863 def writeAtAutoNodes(self, event=None): # pragma: no cover
864 """Write all @auto nodes in the selected outline."""
865 at, c, p = self, self.c, self.c.p
866 c.init_error_dialogs()
867 after, found = p.nodeAfterTree(), False
868 while p and p != after:
869 if p.isAtAutoNode() and not p.isAtIgnoreNode():
870 ok = at.writeOneAtAutoNode(p)
871 if ok:
872 found = True
873 p.moveToNodeAfterTree()
874 else:
875 p.moveToThreadNext()
876 else:
877 p.moveToThreadNext()
878 if g.unitTesting:
879 return
880 if found:
881 g.es("finished")
882 else:
883 g.es("no @auto nodes in the selected tree")
884 c.raise_error_dialogs(kind='write')
886 #@+node:ekr.20220120072251.1: *6* at.writeDirtyAtAutoNodes
887 @cmd('write-dirty-at-auto-nodes') # pragma: no cover
888 def writeDirtyAtAutoNodes(self, event=None):
889 """Write all dirty @auto nodes in the selected outline."""
890 at, c, p = self, self.c, self.c.p
891 c.init_error_dialogs()
892 after, found = p.nodeAfterTree(), False
893 while p and p != after:
894 if p.isAtAutoNode() and not p.isAtIgnoreNode() and p.isDirty():
895 ok = at.writeOneAtAutoNode(p)
896 if ok:
897 found = True
898 p.moveToNodeAfterTree()
899 else:
900 p.moveToThreadNext()
901 else:
902 p.moveToThreadNext()
903 if g.unitTesting:
904 return
905 if found:
906 g.es("finished")
907 else:
908 g.es("no dirty @auto nodes in the selected tree")
909 c.raise_error_dialogs(kind='write')
910 #@+node:ekr.20080711093251.3: *6* at.writeAtShadowNodes
911 @cmd('write-at-shadow-nodes')
912 def writeAtShadowNodes(self, event=None): # pragma: no cover
913 """Write all @shadow nodes in the selected outline."""
914 at, c, p = self, self.c, self.c.p
915 c.init_error_dialogs()
916 after, found = p.nodeAfterTree(), False
917 while p and p != after:
918 if p.atShadowFileNodeName() and not p.isAtIgnoreNode():
919 ok = at.writeOneAtShadowNode(p)
920 if ok:
921 found = True
922 g.blue(f"wrote {p.atShadowFileNodeName()}")
923 p.moveToNodeAfterTree()
924 else:
925 p.moveToThreadNext()
926 else:
927 p.moveToThreadNext()
928 if g.unitTesting:
929 return found
930 if found:
931 g.es("finished")
932 else:
933 g.es("no @shadow nodes in the selected tree")
934 c.raise_error_dialogs(kind='write')
935 return found
937 #@+node:ekr.20220120072917.1: *6* at.writeDirtyAtShadowNodes
938 @cmd('write-dirty-at-shadow-nodes')
939 def writeDirtyAtShadowNodes(self, event=None): # pragma: no cover
940 """Write all @shadow nodes in the selected outline."""
941 at, c, p = self, self.c, self.c.p
942 c.init_error_dialogs()
943 after, found = p.nodeAfterTree(), False
944 while p and p != after:
945 if p.atShadowFileNodeName() and not p.isAtIgnoreNode() and p.isDirty():
946 ok = at.writeOneAtShadowNode(p)
947 if ok:
948 found = True
949 g.blue(f"wrote {p.atShadowFileNodeName()}")
950 p.moveToNodeAfterTree()
951 else:
952 p.moveToThreadNext()
953 else:
954 p.moveToThreadNext()
955 if g.unitTesting:
956 return found
957 if found:
958 g.es("finished")
959 else:
960 g.es("no dirty @shadow nodes in the selected tree")
961 c.raise_error_dialogs(kind='write')
962 return found
964 #@+node:ekr.20041005105605.157: *5* at.putFile
965 def putFile(self, root, fromString='', sentinels=True):
966 """Write the contents of the file to the output stream."""
967 at = self
968 s = fromString if fromString else root.v.b
969 root.clearAllVisitedInTree()
970 at.putAtFirstLines(s)
971 at.putOpenLeoSentinel("@+leo-ver=5")
972 at.putInitialComment()
973 at.putOpenNodeSentinel(root)
974 at.putBody(root, fromString=fromString)
975 # The -leo sentinel is required to handle @last.
976 at.putSentinel("@-leo")
977 root.setVisited()
978 at.putAtLastLines(s)
979 #@+node:ekr.20041005105605.147: *5* at.writeAll & helpers
980 def writeAll(self, all=False, dirty=False):
981 """Write @file nodes in all or part of the outline"""
982 at = self
983 # This is the *only* place where these are set.
984 # promptForDangerousWrite sets cancelFlag only if canCancelFlag is True.
985 at.unchangedFiles = 0
986 at.canCancelFlag = True
987 at.cancelFlag = False
988 at.yesToAll = False
989 files, root = at.findFilesToWrite(all)
990 for p in files:
991 try:
992 at.writeAllHelper(p, root)
993 except Exception: # pragma: no cover
994 at.internalWriteError(p)
995 # Make *sure* these flags are cleared for other commands.
996 at.canCancelFlag = False
997 at.cancelFlag = False
998 at.yesToAll = False
999 # Say the command is finished.
1000 at.reportEndOfWrite(files, all, dirty)
1001 # #2338: Never call at.saveOutlineIfPossible().
1002 #@+node:ekr.20190108052043.1: *6* at.findFilesToWrite
1003 def findFilesToWrite(self, force): # pragma: no cover
1004 """
1005 Return a list of files to write.
1006 We must do this in a prepass, so as to avoid errors later.
1007 """
1008 trace = 'save' in g.app.debug and not g.unitTesting
1009 if trace:
1010 g.trace(f"writing *{'selected' if force else 'all'}* files")
1011 c = self.c
1012 if force:
1013 # The Write @<file> Nodes command.
1014 # Write all nodes in the selected tree.
1015 root = c.p
1016 p = c.p
1017 after = p.nodeAfterTree()
1018 else:
1019 # Write dirty nodes in the entire outline.
1020 root = c.rootPosition()
1021 p = c.rootPosition()
1022 after = None
1023 seen = set()
1024 files = []
1025 while p and p != after:
1026 if p.isAtIgnoreNode() and not p.isAtAsisFileNode():
1027 # Honor @ignore in *body* text, but *not* in @asis nodes.
1028 if p.isAnyAtFileNode():
1029 c.ignored_at_file_nodes.append(p.h)
1030 p.moveToNodeAfterTree()
1031 elif p.isAnyAtFileNode():
1032 data = p.v, g.fullPath(c, p)
1033 if data in seen:
1034 if trace and force:
1035 g.trace('Already seen', p.h)
1036 else:
1037 seen.add(data)
1038 files.append(p.copy())
1039 # Don't scan nested trees???
1040 p.moveToNodeAfterTree()
1041 else:
1042 p.moveToThreadNext()
1043 # When scanning *all* nodes, we only actually write dirty nodes.
1044 if not force:
1045 files = [z for z in files if z.isDirty()]
1046 if trace:
1047 g.printObj([z.h for z in files], tag='Files to be saved')
1048 return files, root
1049 #@+node:ekr.20190108053115.1: *6* at.internalWriteError
1050 def internalWriteError(self, p): # pragma: no cover
1051 """
1052 Fix bug 1260415: https://bugs.launchpad.net/leo-editor/+bug/1260415
1053 Give a more urgent, more specific, more helpful message.
1054 """
1055 g.es_exception()
1056 g.es(f"Internal error writing: {p.h}", color='red')
1057 g.es('Please report this error to:', color='blue')
1058 g.es('https://groups.google.com/forum/#!forum/leo-editor', color='blue')
1059 g.es('Warning: changes to this file will be lost', color='red')
1060 g.es('unless you can save the file successfully.', color='red')
1061 #@+node:ekr.20190108112519.1: *6* at.reportEndOfWrite
1062 def reportEndOfWrite(self, files, all, dirty): # pragma: no cover
1064 at = self
1065 if g.unitTesting:
1066 return
1067 if files:
1068 n = at.unchangedFiles
1069 g.es(f"finished: {n} unchanged file{g.plural(n)}")
1070 elif all:
1071 g.warning("no @<file> nodes in the selected tree")
1072 elif dirty:
1073 g.es("no dirty @<file> nodes in the selected tree")
1074 #@+node:ekr.20041005105605.149: *6* at.writeAllHelper & helper
1075 def writeAllHelper(self, p, root):
1076 """
1077 Write one file for at.writeAll.
1079 Do *not* write @auto files unless p == root.
1081 This prevents the write-all command from needlessly updating
1082 the @persistence data, thereby annoyingly changing the .leo file.
1083 """
1084 at = self
1085 at.root = root
1086 if p.isAtIgnoreNode(): # pragma: no cover
1087 # Should have been handled in findFilesToWrite.
1088 g.trace(f"Can not happen: {p.h} is an @ignore node")
1089 return
1090 try:
1091 at.writePathChanged(p)
1092 except IOError: # pragma: no cover
1093 return
1094 table = (
1095 (p.isAtAsisFileNode, at.asisWrite),
1096 (p.isAtAutoNode, at.writeOneAtAutoNode),
1097 (p.isAtCleanNode, at.writeOneAtCleanNode),
1098 (p.isAtEditNode, at.writeOneAtEditNode),
1099 (p.isAtFileNode, at.writeOneAtFileNode),
1100 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1101 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1102 (p.isAtThinFileNode, at.writeOneAtFileNode),
1103 )
1104 for pred, func in table:
1105 if pred():
1106 func(p) # type:ignore
1107 break
1108 else: # pragma: no cover
1109 g.trace(f"Can not happen: {p.h}")
1110 return
1111 #
1112 # Clear the dirty bits in all descendant nodes.
1113 # The persistence data may still have to be written.
1114 for p2 in p.self_and_subtree(copy=False):
1115 p2.v.clearDirty()
1116 #@+node:ekr.20190108105509.1: *7* at.writePathChanged
1117 def writePathChanged(self, p): # pragma: no cover
1118 """
1119 raise IOError if p's path has changed *and* user forbids the write.
1120 """
1121 at, c = self, self.c
1122 #
1123 # Suppress this message during save-as and save-to commands.
1124 if c.ignoreChangedPaths:
1125 return # pragma: no cover
1126 oldPath = g.os_path_normcase(at.getPathUa(p))
1127 newPath = g.os_path_normcase(g.fullPath(c, p))
1128 try: # #1367: samefile can throw an exception.
1129 changed = oldPath and not os.path.samefile(oldPath, newPath)
1130 except Exception:
1131 changed = True
1132 if not changed:
1133 return
1134 ok = at.promptForDangerousWrite(
1135 fileName=None,
1136 message=(
1137 f"{g.tr('path changed for %s' % (p.h))}\n"
1138 f"{g.tr('write this file anyway?')}"
1139 ),
1140 )
1141 if not ok:
1142 raise IOError
1143 at.setPathUa(p, newPath) # Remember that we have changed paths.
1144 #@+node:ekr.20190109172025.1: *5* at.writeAtAutoContents
1145 def writeAtAutoContents(self, fileName, root): # pragma: no cover
1146 """Common helper for atAutoToString and writeOneAtAutoNode."""
1147 at, c = self, self.c
1148 # Dispatch the proper writer.
1149 junk, ext = g.os_path_splitext(fileName)
1150 writer = at.dispatch(ext, root)
1151 if writer:
1152 at.outputList = []
1153 writer(root)
1154 return '' if at.errors else ''.join(at.outputList)
1155 if root.isAtAutoRstNode():
1156 # An escape hatch: fall back to the theRst writer
1157 # if there is no rst writer plugin.
1158 at.outputFile = outputFile = io.StringIO()
1159 ok = c.rstCommands.writeAtAutoFile(root, fileName, outputFile)
1160 return outputFile.close() if ok else None
1161 # leo 5.6: allow undefined section references in all @auto files.
1162 ivar = 'allow_undefined_refs'
1163 try:
1164 setattr(at, ivar, True)
1165 at.outputList = []
1166 at.putFile(root, sentinels=False)
1167 return '' if at.errors else ''.join(at.outputList)
1168 except Exception:
1169 return None
1170 finally:
1171 if hasattr(at, ivar):
1172 delattr(at, ivar)
1173 #@+node:ekr.20190111153522.1: *5* at.writeX...
1174 #@+node:ekr.20041005105605.154: *6* at.asisWrite & helper
1175 def asisWrite(self, root): # pragma: no cover
1176 at, c = self, self.c
1177 try:
1178 c.endEditing()
1179 c.init_error_dialogs()
1180 fileName = at.initWriteIvars(root)
1181 # #1450.
1182 if not fileName or not at.precheck(fileName, root):
1183 at.addToOrphanList(root)
1184 return
1185 at.outputList = []
1186 for p in root.self_and_subtree(copy=False):
1187 at.writeAsisNode(p)
1188 if not at.errors:
1189 contents = ''.join(at.outputList)
1190 at.replaceFile(contents, at.encoding, fileName, root)
1191 except Exception:
1192 at.writeException(fileName, root)
1194 silentWrite = asisWrite # Compatibility with old scripts.
1195 #@+node:ekr.20170331141933.1: *7* at.writeAsisNode
1196 def writeAsisNode(self, p): # pragma: no cover
1197 """Write the p's node to an @asis file."""
1198 at = self
1200 def put(s):
1201 """Append s to self.output_list."""
1202 # #1480: Avoid calling at.os().
1203 s = g.toUnicode(s, at.encoding, reportErrors=True)
1204 at.outputList.append(s)
1206 # Write the headline only if it starts with '@@'.
1208 s = p.h
1209 if g.match(s, 0, "@@"):
1210 s = s[2:]
1211 if s:
1212 put('\n') # Experimental.
1213 put(s)
1214 put('\n')
1215 # Write the body.
1216 s = p.b
1217 if s:
1218 put(s)
1219 #@+node:ekr.20041005105605.151: *6* at.writeMissing & helper
1220 def writeMissing(self, p): # pragma: no cover
1221 at, c = self, self.c
1222 writtenFiles = False
1223 c.init_error_dialogs()
1224 # #1450.
1225 at.initWriteIvars(root=p.copy())
1226 p = p.copy()
1227 after = p.nodeAfterTree()
1228 while p and p != after: # Don't use iterator.
1229 if (
1230 p.isAtAsisFileNode() or (p.isAnyAtFileNode() and not p.isAtIgnoreNode())
1231 ):
1232 fileName = p.anyAtFileNodeName()
1233 if fileName:
1234 fileName = g.fullPath(c, p) # #1914.
1235 if at.precheck(fileName, p):
1236 at.writeMissingNode(p)
1237 writtenFiles = True
1238 else:
1239 at.addToOrphanList(p)
1240 p.moveToNodeAfterTree()
1241 elif p.isAtIgnoreNode():
1242 p.moveToNodeAfterTree()
1243 else:
1244 p.moveToThreadNext()
1245 if not g.unitTesting:
1246 if writtenFiles > 0:
1247 g.es("finished")
1248 else:
1249 g.es("no @file node in the selected tree")
1250 c.raise_error_dialogs(kind='write')
1251 #@+node:ekr.20041005105605.152: *7* at.writeMissingNode
1252 def writeMissingNode(self, p): # pragma: no cover
1254 at = self
1255 table = (
1256 (p.isAtAsisFileNode, at.asisWrite),
1257 (p.isAtAutoNode, at.writeOneAtAutoNode),
1258 (p.isAtCleanNode, at.writeOneAtCleanNode),
1259 (p.isAtEditNode, at.writeOneAtEditNode),
1260 (p.isAtFileNode, at.writeOneAtFileNode),
1261 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1262 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1263 (p.isAtThinFileNode, at.writeOneAtFileNode),
1264 )
1265 for pred, func in table:
1266 if pred():
1267 func(p) # type:ignore
1268 return
1269 g.trace(f"Can not happen unknown @<file> kind: {p.h}")
1270 #@+node:ekr.20070806141607: *6* at.writeOneAtAutoNode & helpers
1271 def writeOneAtAutoNode(self, p): # pragma: no cover
1272 """
1273 Write p, an @auto node.
1274 File indices *must* have already been assigned.
1275 Return True if the node was written successfully.
1276 """
1277 at, c = self, self.c
1278 root = p.copy()
1279 try:
1280 c.endEditing()
1281 if not p.atAutoNodeName():
1282 return False
1283 fileName = at.initWriteIvars(root)
1284 at.sentinels = False
1285 # #1450.
1286 if not fileName or not at.precheck(fileName, root):
1287 at.addToOrphanList(root)
1288 return False
1289 if c.persistenceController:
1290 c.persistenceController.update_before_write_foreign_file(root)
1291 contents = at.writeAtAutoContents(fileName, root)
1292 if contents is None:
1293 g.es("not written:", fileName)
1294 at.addToOrphanList(root)
1295 return False
1296 at.replaceFile(contents, at.encoding, fileName, root,
1297 ignoreBlankLines=root.isAtAutoRstNode())
1298 return True
1299 except Exception:
1300 at.writeException(fileName, root)
1301 return False
1302 #@+node:ekr.20140728040812.17993: *7* at.dispatch & helpers
1303 def dispatch(self, ext, p): # pragma: no cover
1304 """Return the correct writer function for p, an @auto node."""
1305 at = self
1306 # Match @auto type before matching extension.
1307 return at.writer_for_at_auto(p) or at.writer_for_ext(ext)
1308 #@+node:ekr.20140728040812.17995: *8* at.writer_for_at_auto
1309 def writer_for_at_auto(self, root): # pragma: no cover
1310 """A factory returning a writer function for the given kind of @auto directive."""
1311 at = self
1312 d = g.app.atAutoWritersDict
1313 for key in d:
1314 aClass = d.get(key)
1315 if aClass and g.match_word(root.h, 0, key):
1317 def writer_for_at_auto_cb(root):
1318 # pylint: disable=cell-var-from-loop
1319 try:
1320 writer = aClass(at.c)
1321 s = writer.write(root)
1322 return s
1323 except Exception:
1324 g.es_exception()
1325 return None
1327 return writer_for_at_auto_cb
1328 return None
1329 #@+node:ekr.20140728040812.17997: *8* at.writer_for_ext
1330 def writer_for_ext(self, ext): # pragma: no cover
1331 """A factory returning a writer function for the given file extension."""
1332 at = self
1333 d = g.app.writersDispatchDict
1334 aClass = d.get(ext)
1335 if aClass:
1337 def writer_for_ext_cb(root):
1338 try:
1339 return aClass(at.c).write(root)
1340 except Exception:
1341 g.es_exception()
1342 return None
1344 return writer_for_ext_cb
1346 return None
1347 #@+node:ekr.20210501064359.1: *6* at.writeOneAtCleanNode
1348 def writeOneAtCleanNode(self, root): # pragma: no cover
1349 """Write one @clean file..
1350 root is the position of an @clean node.
1351 """
1352 at, c = self, self.c
1353 try:
1354 c.endEditing()
1355 fileName = at.initWriteIvars(root)
1356 at.sentinels = False
1357 if not fileName or not at.precheck(fileName, root):
1358 return
1359 at.outputList = []
1360 at.putFile(root, sentinels=False)
1361 at.warnAboutOrphandAndIgnoredNodes()
1362 if at.errors:
1363 g.es("not written:", g.shortFileName(fileName))
1364 at.addToOrphanList(root)
1365 else:
1366 contents = ''.join(at.outputList)
1367 at.replaceFile(contents, at.encoding, fileName, root)
1368 except Exception:
1369 at.writeException(fileName, root)
1370 #@+node:ekr.20090225080846.5: *6* at.writeOneAtEditNode
1371 def writeOneAtEditNode(self, p): # pragma: no cover
1372 """Write one @edit node."""
1373 at, c = self, self.c
1374 root = p.copy()
1375 try:
1376 c.endEditing()
1377 c.init_error_dialogs()
1378 if not p.atEditNodeName():
1379 return False
1380 if p.hasChildren():
1381 g.error('@edit nodes must not have children')
1382 g.es('To save your work, convert @edit to @auto, @file or @clean')
1383 return False
1384 fileName = at.initWriteIvars(root)
1385 at.sentinels = False
1386 # #1450.
1387 if not fileName or not at.precheck(fileName, root):
1388 at.addToOrphanList(root)
1389 return False
1390 contents = ''.join([s for s in g.splitLines(p.b)
1391 if at.directiveKind4(s, 0) == at.noDirective])
1392 at.replaceFile(contents, at.encoding, fileName, root)
1393 c.raise_error_dialogs(kind='write')
1394 return True
1395 except Exception:
1396 at.writeException(fileName, root)
1397 return False
1398 #@+node:ekr.20210501075610.1: *6* at.writeOneAtFileNode
1399 def writeOneAtFileNode(self, root): # pragma: no cover
1400 """Write @file or @thin file."""
1401 at, c = self, self.c
1402 try:
1403 c.endEditing()
1404 fileName = at.initWriteIvars(root)
1405 at.sentinels = True
1406 if not fileName or not at.precheck(fileName, root):
1407 # Raise dialog warning of data loss.
1408 at.addToOrphanList(root)
1409 return
1410 at.outputList = []
1411 at.putFile(root, sentinels=True)
1412 at.warnAboutOrphandAndIgnoredNodes()
1413 if at.errors:
1414 g.es("not written:", g.shortFileName(fileName))
1415 at.addToOrphanList(root)
1416 else:
1417 contents = ''.join(at.outputList)
1418 at.replaceFile(contents, at.encoding, fileName, root)
1419 except Exception:
1420 at.writeException(fileName, root)
1421 #@+node:ekr.20210501065352.1: *6* at.writeOneAtNosentNode
1422 def writeOneAtNosentNode(self, root): # pragma: no cover
1423 """Write one @nosent node.
1424 root is the position of an @<file> node.
1425 sentinels will be False for @clean and @nosent nodes.
1426 """
1427 at, c = self, self.c
1428 try:
1429 c.endEditing()
1430 fileName = at.initWriteIvars(root)
1431 at.sentinels = False
1432 if not fileName or not at.precheck(fileName, root):
1433 return
1434 at.outputList = []
1435 at.putFile(root, sentinels=False)
1436 at.warnAboutOrphandAndIgnoredNodes()
1437 if at.errors:
1438 g.es("not written:", g.shortFileName(fileName))
1439 at.addToOrphanList(root)
1440 else:
1441 contents = ''.join(at.outputList)
1442 at.replaceFile(contents, at.encoding, fileName, root)
1443 except Exception:
1444 at.writeException(fileName, root)
1445 #@+node:ekr.20080711093251.5: *6* at.writeOneAtShadowNode & helper
1446 def writeOneAtShadowNode(self, p, testing=False): # pragma: no cover
1447 """
1448 Write p, an @shadow node.
1449 File indices *must* have already been assigned.
1451 testing: set by unit tests to suppress the call to at.precheck.
1452 Testing is not the same as g.unitTesting.
1453 """
1454 at, c = self, self.c
1455 root = p.copy()
1456 x = c.shadowController
1457 try:
1458 c.endEditing() # Capture the current headline.
1459 fn = p.atShadowFileNodeName()
1460 assert fn, p.h
1461 self.adjustTargetLanguage(fn)
1462 # A hack to support unknown extensions. May set c.target_language.
1463 full_path = g.fullPath(c, p)
1464 at.initWriteIvars(root)
1465 # Force python sentinels to suppress an error message.
1466 # The actual sentinels will be set below.
1467 at.endSentinelComment = None
1468 at.startSentinelComment = "#"
1469 # Make sure we can compute the shadow directory.
1470 private_fn = x.shadowPathName(full_path)
1471 if not private_fn:
1472 return False
1473 if not testing and not at.precheck(full_path, root):
1474 return False
1475 #
1476 # Bug fix: Leo 4.5.1:
1477 # use x.markerFromFileName to force the delim to match
1478 # what is used in x.propegate changes.
1479 marker = x.markerFromFileName(full_path)
1480 at.startSentinelComment, at.endSentinelComment = marker.getDelims()
1481 if g.unitTesting:
1482 ivars_dict = g.getIvarsDict(at)
1483 #
1484 # Write the public and private files to strings.
1486 def put(sentinels):
1487 at.outputList = []
1488 at.sentinels = sentinels
1489 at.putFile(root, sentinels=sentinels)
1490 return '' if at.errors else ''.join(at.outputList)
1492 at.public_s = put(False)
1493 at.private_s = put(True)
1494 at.warnAboutOrphandAndIgnoredNodes()
1495 if g.unitTesting:
1496 exceptions = ('public_s', 'private_s', 'sentinels', 'outputList')
1497 assert g.checkUnchangedIvars(
1498 at, ivars_dict, exceptions), 'writeOneAtShadowNode'
1499 if not at.errors:
1500 # Write the public and private files.
1501 x.makeShadowDirectory(full_path)
1502 # makeShadowDirectory takes a *public* file name.
1503 x.replaceFileWithString(at.encoding, private_fn, at.private_s)
1504 x.replaceFileWithString(at.encoding, full_path, at.public_s)
1505 at.checkPythonCode(contents=at.private_s, fileName=full_path, root=root)
1506 if at.errors:
1507 g.error("not written:", full_path)
1508 at.addToOrphanList(root)
1509 else:
1510 root.clearDirty()
1511 return not at.errors
1512 except Exception:
1513 at.writeException(full_path, root)
1514 return False
1515 #@+node:ekr.20080819075811.13: *7* at.adjustTargetLanguage
1516 def adjustTargetLanguage(self, fn): # pragma: no cover
1517 """Use the language implied by fn's extension if
1518 there is a conflict between it and c.target_language."""
1519 at = self
1520 c = at.c
1521 junk, ext = g.os_path_splitext(fn)
1522 if ext:
1523 if ext.startswith('.'):
1524 ext = ext[1:]
1525 language = g.app.extension_dict.get(ext)
1526 if language:
1527 c.target_language = language
1528 else:
1529 # An unknown language.
1530 # Use the default language, **not** 'unknown_language'
1531 pass
1532 #@+node:ekr.20190111153506.1: *5* at.XToString
1533 #@+node:ekr.20190109160056.1: *6* at.atAsisToString
1534 def atAsisToString(self, root): # pragma: no cover
1535 """Write the @asis node to a string."""
1536 at, c = self, self.c
1537 try:
1538 c.endEditing()
1539 fileName = at.initWriteIvars(root)
1540 at.outputList = []
1541 for p in root.self_and_subtree(copy=False):
1542 at.writeAsisNode(p)
1543 return '' if at.errors else ''.join(at.outputList)
1544 except Exception:
1545 at.writeException(fileName, root)
1546 return ''
1547 #@+node:ekr.20190109160056.2: *6* at.atAutoToString
1548 def atAutoToString(self, root): # pragma: no cover
1549 """Write the root @auto node to a string, and return it."""
1550 at, c = self, self.c
1551 try:
1552 c.endEditing()
1553 fileName = at.initWriteIvars(root)
1554 at.sentinels = False
1555 # #1450.
1556 if not fileName:
1557 at.addToOrphanList(root)
1558 return ''
1559 return at.writeAtAutoContents(fileName, root) or ''
1560 except Exception:
1561 at.writeException(fileName, root)
1562 return ''
1563 #@+node:ekr.20190109160056.3: *6* at.atEditToString
1564 def atEditToString(self, root): # pragma: no cover
1565 """Write one @edit node."""
1566 at, c = self, self.c
1567 try:
1568 c.endEditing()
1569 if root.hasChildren():
1570 g.error('@edit nodes must not have children')
1571 g.es('To save your work, convert @edit to @auto, @file or @clean')
1572 return False
1573 fileName = at.initWriteIvars(root)
1574 at.sentinels = False
1575 # #1450.
1576 if not fileName:
1577 at.addToOrphanList(root)
1578 return ''
1579 contents = ''.join([
1580 s for s in g.splitLines(root.b)
1581 if at.directiveKind4(s, 0) == at.noDirective])
1582 return contents
1583 except Exception:
1584 at.writeException(fileName, root)
1585 return ''
1586 #@+node:ekr.20190109142026.1: *6* at.atFileToString
1587 def atFileToString(self, root, sentinels=True): # pragma: no cover
1588 """Write an external file to a string, and return its contents."""
1589 at, c = self, self.c
1590 try:
1591 c.endEditing()
1592 at.initWriteIvars(root)
1593 at.sentinels = sentinels
1594 at.outputList = []
1595 at.putFile(root, sentinels=sentinels)
1596 assert root == at.root, 'write'
1597 contents = '' if at.errors else ''.join(at.outputList)
1598 return contents
1599 except Exception:
1600 at.exception("exception preprocessing script")
1601 root.v._p_changed = True
1602 return ''
1603 #@+node:ekr.20050506084734: *6* at.stringToString
1604 def stringToString(self, root, s, forcePythonSentinels=True, sentinels=True): # pragma: no cover
1605 """
1606 Write an external file from a string.
1608 This is at.write specialized for scripting.
1609 """
1610 at, c = self, self.c
1611 try:
1612 c.endEditing()
1613 at.initWriteIvars(root)
1614 if forcePythonSentinels:
1615 at.endSentinelComment = None
1616 at.startSentinelComment = "#"
1617 at.language = "python"
1618 at.sentinels = sentinels
1619 at.outputList = []
1620 at.putFile(root, fromString=s, sentinels=sentinels)
1621 contents = '' if at.errors else ''.join(at.outputList)
1622 # Major bug: failure to clear this wipes out headlines!
1623 # Sometimes this causes slight problems...
1624 if root:
1625 root.v._p_changed = True
1626 return contents
1627 except Exception:
1628 at.exception("exception preprocessing script")
1629 return ''
1630 #@+node:ekr.20041005105605.160: *4* Writing helpers
1631 #@+node:ekr.20041005105605.161: *5* at.putBody & helper
1632 def putBody(self, p, fromString=''):
1633 """
1634 Generate the body enclosed in sentinel lines.
1635 Return True if the body contains an @others line.
1636 """
1637 at = self
1638 #
1639 # New in 4.3 b2: get s from fromString if possible.
1640 s = fromString if fromString else p.b
1641 p.v.setVisited()
1642 # Make sure v is never expanded again.
1643 # Suppress orphans check.
1644 #
1645 # #1048 & #1037: regularize most trailing whitespace.
1646 if s and (at.sentinels or at.force_newlines_in_at_nosent_bodies):
1647 if not s.endswith('\n'):
1648 s = s + '\n'
1651 class Status:
1652 at_comment_seen = False
1653 at_delims_seen = False
1654 at_warning_given = False
1655 has_at_others = False
1656 in_code = True
1659 i = 0
1660 status = Status()
1661 while i < len(s):
1662 next_i = g.skip_line(s, i)
1663 assert next_i > i, 'putBody'
1664 kind = at.directiveKind4(s, i)
1665 at.putLine(i, kind, p, s, status)
1666 i = next_i
1667 if not status.in_code:
1668 at.putEndDocLine()
1669 return status.has_at_others
1670 #@+node:ekr.20041005105605.163: *6* at.putLine
1671 def putLine(self, i, kind, p, s, status):
1672 """Put the line at s[i:] of the given kind, updating the status."""
1673 at = self
1674 if kind == at.noDirective:
1675 if status.in_code:
1676 # Important: the so-called "name" must include brackets.
1677 name, n1, n2 = at.findSectionName(s, i, p)
1678 if name:
1679 at.putRefLine(s, i, n1, n2, name, p)
1680 else:
1681 at.putCodeLine(s, i)
1682 else:
1683 at.putDocLine(s, i)
1684 elif kind in (at.docDirective, at.atDirective):
1685 if not status.in_code:
1686 # Bug fix 12/31/04: handle adjacent doc parts.
1687 at.putEndDocLine()
1688 at.putStartDocLine(s, i, kind)
1689 status.in_code = False
1690 elif kind in (at.cDirective, at.codeDirective):
1691 # Only @c and @code end a doc part.
1692 if not status.in_code:
1693 at.putEndDocLine()
1694 at.putDirective(s, i, p)
1695 status.in_code = True
1696 elif kind == at.allDirective:
1697 if status.in_code:
1698 if p == self.root:
1699 at.putAtAllLine(s, i, p)
1700 else:
1701 at.error(f"@all not valid in: {p.h}") # pragma: no cover
1702 else:
1703 at.putDocLine(s, i)
1704 elif kind == at.othersDirective:
1705 if status.in_code:
1706 if status.has_at_others:
1707 at.error(f"multiple @others in: {p.h}") # pragma: no cover
1708 else:
1709 at.putAtOthersLine(s, i, p)
1710 status.has_at_others = True
1711 else:
1712 at.putDocLine(s, i)
1713 elif kind == at.startVerbatim: # pragma: no cover
1714 # Fix bug 778204: @verbatim not a valid Leo directive.
1715 if g.unitTesting:
1716 # A hack: unit tests for @shadow use @verbatim as a kind of directive.
1717 pass
1718 else:
1719 at.error(f"@verbatim is not a Leo directive: {p.h}")
1720 elif kind == at.miscDirective:
1721 # Fix bug 583878: Leo should warn about @comment/@delims clashes.
1722 if g.match_word(s, i, '@comment'):
1723 status.at_comment_seen = True
1724 elif g.match_word(s, i, '@delims'):
1725 status.at_delims_seen = True
1726 if (
1727 status.at_comment_seen and
1728 status.at_delims_seen and not
1729 status.at_warning_given
1730 ): # pragma: no cover
1731 status.at_warning_given = True
1732 at.error(f"@comment and @delims in node {p.h}")
1733 at.putDirective(s, i, p)
1734 else:
1735 at.error(f"putBody: can not happen: unknown directive kind: {kind}") # pragma: no cover
1736 #@+node:ekr.20041005105605.164: *5* writing code lines...
1737 #@+node:ekr.20041005105605.165: *6* at: @all
1738 #@+node:ekr.20041005105605.166: *7* at.putAtAllLine
1739 def putAtAllLine(self, s, i, p):
1740 """Put the expansion of @all."""
1741 at = self
1742 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1743 k = g.skip_to_end_of_line(s, i)
1744 at.putLeadInSentinel(s, i, j)
1745 at.indent += delta
1746 at.putSentinel("@+" + s[j + 1 : k].strip())
1747 # s[j:k] starts with '@all'
1748 for child in p.children():
1749 at.putAtAllChild(child)
1750 at.putSentinel("@-all")
1751 at.indent -= delta
1752 #@+node:ekr.20041005105605.167: *7* at.putAtAllBody
1753 def putAtAllBody(self, p):
1754 """ Generate the body enclosed in sentinel lines."""
1755 at = self
1756 s = p.b
1757 p.v.setVisited()
1758 # Make sure v is never expanded again.
1759 # Suppress orphans check.
1760 if at.sentinels and s and s[-1] != '\n':
1761 s = s + '\n'
1762 i = 0
1763 # Leo 6.6. This code never changes at.in_code status!
1764 while i < len(s):
1765 next_i = g.skip_line(s, i)
1766 assert next_i > i
1767 at.putCodeLine(s, i)
1768 i = next_i
1769 #@+node:ekr.20041005105605.169: *7* at.putAtAllChild
1770 def putAtAllChild(self, p):
1771 """
1772 This code puts only the first of two or more cloned siblings, preceding
1773 the clone with an @clone n sentinel.
1775 This is a debatable choice: the cloned tree appears only once in the
1776 external file. This should be benign; the text created by @all is
1777 likely to be used only for recreating the outline in Leo. The
1778 representation in the derived file doesn't matter much.
1779 """
1780 at = self
1781 at.putOpenNodeSentinel(p, inAtAll=True)
1782 # Suppress warnings about @file nodes.
1783 at.putAtAllBody(p)
1784 for child in p.children():
1785 at.putAtAllChild(child) # pragma: no cover (recursive call)
1786 #@+node:ekr.20041005105605.170: *6* at: @others
1787 #@+node:ekr.20041005105605.173: *7* at.putAtOthersLine & helper
1788 def putAtOthersLine(self, s, i, p):
1789 """Put the expansion of @others."""
1790 at = self
1791 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1792 k = g.skip_to_end_of_line(s, i)
1793 at.putLeadInSentinel(s, i, j)
1794 at.indent += delta
1795 # s[j:k] starts with '@others'
1796 # Never write lws in new sentinels.
1797 at.putSentinel("@+" + s[j + 1 : k].strip())
1798 for child in p.children():
1799 p = child.copy()
1800 after = p.nodeAfterTree()
1801 while p and p != after:
1802 if at.validInAtOthers(p):
1803 at.putOpenNodeSentinel(p)
1804 at_others_flag = at.putBody(p)
1805 if at_others_flag:
1806 p.moveToNodeAfterTree()
1807 else:
1808 p.moveToThreadNext()
1809 else:
1810 p.moveToNodeAfterTree()
1811 # This is the same in both old and new sentinels.
1812 at.putSentinel("@-others")
1813 at.indent -= delta
1814 #@+node:ekr.20041005105605.171: *8* at.validInAtOthers
1815 def validInAtOthers(self, p):
1816 """
1817 Return True if p should be included in the expansion of the @others
1818 directive in the body text of p's parent.
1819 """
1820 at = self
1821 i = g.skip_ws(p.h, 0)
1822 isSection, junk = at.isSectionName(p.h, i)
1823 if isSection:
1824 return False # A section definition node.
1825 if at.sentinels:
1826 # @ignore must not stop expansion here!
1827 return True
1828 if p.isAtIgnoreNode(): # pragma: no cover
1829 g.error('did not write @ignore node', p.v.h)
1830 return False
1831 return True
1832 #@+node:ekr.20041005105605.199: *6* at.findSectionName
1833 def findSectionName(self, s, i, p):
1834 """
1835 Return n1, n2 representing a section name.
1837 Return the reference, *including* brackes.
1838 """
1839 at = self
1841 def is_space(i1, i2):
1842 """A replacement for s[i1 : i2] that doesn't create any substring."""
1843 return i == j or all(s[z] in ' \t\n' for z in range(i1, i2))
1845 end = s.find('\n', i)
1846 j = len(s) if end == -1 else end
1847 # Careful: don't look beyond the end of the line!
1848 if end == -1:
1849 n1 = s.find(at.section_delim1, i)
1850 n2 = s.find(at.section_delim2, i)
1851 else:
1852 n1 = s.find(at.section_delim1, i, end)
1853 n2 = s.find(at.section_delim2, i, end)
1854 n3 = n2 + len(at.section_delim2)
1855 if -1 < n1 < n2: # A *possible* section reference.
1856 if is_space(i, n1) and is_space(n3, j): # A *real* section reference.
1857 return s[n1:n3], n1, n3
1858 # An apparent section reference.
1859 if 'sections' in g.app.debug and not g.unitTesting: # pragma: no cover
1860 i1, i2 = g.getLine(s, i)
1861 g.es_print('Ignoring apparent section reference:', color='red')
1862 g.es_print('Node: ', p.h)
1863 g.es_print('Line: ', s[i1:i2].rstrip())
1864 return None, 0, 0
1865 #@+node:ekr.20041005105605.174: *6* at.putCodeLine
1866 def putCodeLine(self, s, i):
1867 """Put a normal code line."""
1868 at = self
1869 # Put @verbatim sentinel if required.
1870 k = g.skip_ws(s, i)
1871 if g.match(s, k, self.startSentinelComment + '@'):
1872 self.putSentinel('@verbatim')
1873 j = g.skip_line(s, i)
1874 line = s[i:j]
1875 # Don't put any whitespace in otherwise blank lines.
1876 if len(line) > 1: # Preserve *anything* the user puts on the line!!!
1877 at.putIndent(at.indent, line)
1878 if line[-1:] == '\n':
1879 at.os(line[:-1])
1880 at.onl()
1881 else:
1882 at.os(line)
1883 elif line and line[-1] == '\n':
1884 at.onl()
1885 elif line:
1886 at.os(line) # Bug fix: 2013/09/16
1887 else:
1888 g.trace('Can not happen: completely empty line') # pragma: no cover
1889 #@+node:ekr.20041005105605.176: *6* at.putRefLine
1890 def putRefLine(self, s, i, n1, n2, name, p):
1891 """
1892 Put a line containing one or more references.
1894 Important: the so-called name *must* include brackets.
1895 """
1896 at = self
1897 ref = g.findReference(name, p)
1898 if ref:
1899 junk, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1900 at.putLeadInSentinel(s, i, n1)
1901 at.indent += delta
1902 at.putSentinel("@+" + name)
1903 at.putOpenNodeSentinel(ref)
1904 at.putBody(ref)
1905 at.putSentinel("@-" + name)
1906 at.indent -= delta
1907 return
1908 if hasattr(at, 'allow_undefined_refs'): # pragma: no cover
1909 p.v.setVisited() # #2311
1910 # Allow apparent section reference: just write the line.
1911 at.putCodeLine(s, i)
1912 else: # pragma: no cover
1913 # Do give this error even if unit testing.
1914 at.writeError(
1915 f"undefined section: {g.truncate(name, 60)}\n"
1916 f" referenced from: {g.truncate(p.h, 60)}")
1917 #@+node:ekr.20041005105605.180: *5* writing doc lines...
1918 #@+node:ekr.20041005105605.181: *6* at.putBlankDocLine
1919 def putBlankDocLine(self):
1920 at = self
1921 if not at.endSentinelComment:
1922 at.putIndent(at.indent)
1923 at.os(at.startSentinelComment)
1924 # #1496: Retire the @doc convention.
1925 # Remove the blank.
1926 # at.oblank()
1927 at.onl()
1928 #@+node:ekr.20041005105605.183: *6* at.putDocLine
1929 def putDocLine(self, s, i):
1930 """Handle one line of a doc part."""
1931 at = self
1932 j = g.skip_line(s, i)
1933 s = s[i:j]
1934 #
1935 # #1496: Retire the @doc convention:
1936 # Strip all trailing ws here.
1937 if not s.strip():
1938 # A blank line.
1939 at.putBlankDocLine()
1940 return
1941 # Write the line as it is.
1942 at.putIndent(at.indent)
1943 if not at.endSentinelComment:
1944 at.os(at.startSentinelComment)
1945 # #1496: Retire the @doc convention.
1946 # Leave this blank. The line is not blank.
1947 at.oblank()
1948 at.os(s)
1949 if not s.endswith('\n'):
1950 at.onl() # pragma: no cover
1951 #@+node:ekr.20041005105605.185: *6* at.putEndDocLine
1952 def putEndDocLine(self):
1953 """Write the conclusion of a doc part."""
1954 at = self
1955 # Put the closing delimiter if we are using block comments.
1956 if at.endSentinelComment:
1957 at.putIndent(at.indent)
1958 at.os(at.endSentinelComment)
1959 at.onl() # Note: no trailing whitespace.
1960 #@+node:ekr.20041005105605.182: *6* at.putStartDocLine
1961 def putStartDocLine(self, s, i, kind):
1962 """Write the start of a doc part."""
1963 at = self
1964 sentinel = "@+doc" if kind == at.docDirective else "@+at"
1965 directive = "@doc" if kind == at.docDirective else "@"
1966 # Put whatever follows the directive in the sentinel.
1967 # Skip past the directive.
1968 i += len(directive)
1969 j = g.skip_to_end_of_line(s, i)
1970 follow = s[i:j]
1971 # Put the opening @+doc or @-doc sentinel, including whatever follows the directive.
1972 at.putSentinel(sentinel + follow)
1973 # Put the opening comment if we are using block comments.
1974 if at.endSentinelComment:
1975 at.putIndent(at.indent)
1976 at.os(at.startSentinelComment)
1977 at.onl()
1978 #@+node:ekr.20041005105605.187: *4* Writing sentinels...
1979 #@+node:ekr.20041005105605.188: *5* at.nodeSentinelText & helper
1980 def nodeSentinelText(self, p):
1981 """Return the text of a @+node or @-node sentinel for p."""
1982 at = self
1983 h = at.removeCommentDelims(p)
1984 if getattr(at, 'at_shadow_test_hack', False): # pragma: no cover
1985 # A hack for @shadow unit testing.
1986 # see AtShadowTestCase.makePrivateLines.
1987 return h
1988 gnx = p.v.fileIndex
1989 level = 1 + p.level() - self.root.level()
1990 if level > 2:
1991 return f"{gnx}: *{level}* {h}"
1992 return f"{gnx}: {'*' * level} {h}"
1993 #@+node:ekr.20041005105605.189: *6* at.removeCommentDelims
1994 def removeCommentDelims(self, p):
1995 """
1996 If the present @language/@comment settings do not specify a single-line comment
1997 we remove all block comment delims from h. This prevents headline text from
1998 interfering with the parsing of node sentinels.
1999 """
2000 at = self
2001 start = at.startSentinelComment
2002 end = at.endSentinelComment
2003 h = p.h
2004 if end:
2005 h = h.replace(start, "")
2006 h = h.replace(end, "")
2007 return h
2008 #@+node:ekr.20041005105605.190: *5* at.putLeadInSentinel
2009 def putLeadInSentinel(self, s, i, j):
2010 """
2011 Set at.leadingWs as needed for @+others and @+<< sentinels.
2013 i points at the start of a line.
2014 j points at @others or a section reference.
2015 """
2016 at = self
2017 at.leadingWs = "" # Set the default.
2018 if i == j:
2019 return # The @others or ref starts a line.
2020 k = g.skip_ws(s, i)
2021 if j == k:
2022 # Remember the leading whitespace, including its spelling.
2023 at.leadingWs = s[i:j]
2024 else:
2025 self.putIndent(at.indent) # 1/29/04: fix bug reported by Dan Winkler.
2026 at.os(s[i:j])
2027 at.onl_sent()
2028 #@+node:ekr.20041005105605.192: *5* at.putOpenLeoSentinel 4.x
2029 def putOpenLeoSentinel(self, s):
2030 """Write @+leo sentinel."""
2031 at = self
2032 if at.sentinels or hasattr(at, 'force_sentinels'):
2033 s = s + "-thin"
2034 encoding = at.encoding.lower()
2035 if encoding != "utf-8": # pragma: no cover
2036 # New in 4.2: encoding fields end in ",."
2037 s = s + f"-encoding={encoding},."
2038 at.putSentinel(s)
2039 #@+node:ekr.20041005105605.193: *5* at.putOpenNodeSentinel
2040 def putOpenNodeSentinel(self, p, inAtAll=False):
2041 """Write @+node sentinel for p."""
2042 # Note: lineNumbers.py overrides this method.
2043 at = self
2044 if not inAtAll and p.isAtFileNode() and p != at.root: # pragma: no cover
2045 at.writeError("@file not valid in: " + p.h)
2046 return
2047 s = at.nodeSentinelText(p)
2048 at.putSentinel("@+node:" + s)
2049 # Leo 4.7: we never write tnodeLists.
2050 #@+node:ekr.20041005105605.194: *5* at.putSentinel (applies cweb hack) 4.x
2051 def putSentinel(self, s):
2052 """
2053 Write a sentinel whose text is s, applying the CWEB hack if needed.
2055 This method outputs all sentinels.
2056 """
2057 at = self
2058 if at.sentinels or hasattr(at, 'force_sentinels'):
2059 at.putIndent(at.indent)
2060 at.os(at.startSentinelComment)
2061 # #2194. The following would follow the black convention,
2062 # but doing so is a dubious idea.
2063 # at.os(' ')
2064 # Apply the cweb hack to s:
2065 # If the opening comment delim ends in '@',
2066 # double all '@' signs except the first.
2067 start = at.startSentinelComment
2068 if start and start[-1] == '@':
2069 s = s.replace('@', '@@')[1:]
2070 at.os(s)
2071 if at.endSentinelComment:
2072 at.os(at.endSentinelComment)
2073 at.onl()
2074 #@+node:ekr.20041005105605.196: *4* Writing utils...
2075 #@+node:ekr.20181024134823.1: *5* at.addToOrphanList
2076 def addToOrphanList(self, root): # pragma: no cover
2077 """Mark the root as erroneous for c.raise_error_dialogs()."""
2078 c = self.c
2079 # Fix #1050:
2080 root.setOrphan()
2081 c.orphan_at_file_nodes.append(root.h)
2082 #@+node:ekr.20220120210617.1: *5* at.checkPyflakes
2083 def checkPyflakes(self, contents, fileName, root): # pragma: no cover
2084 at = self
2085 ok = True
2086 if g.unitTesting or not at.runPyFlakesOnWrite:
2087 return ok
2088 if not contents or not fileName or not fileName.endswith('.py'):
2089 return ok
2090 ok = self.runPyflakes(root)
2091 if not ok:
2092 g.app.syntax_error_files.append(g.shortFileName(fileName))
2093 return ok
2094 #@+node:ekr.20090514111518.5661: *5* at.checkPythonCode & helpers
2095 def checkPythonCode(self, contents, fileName, root): # pragma: no cover
2096 """Perform python-related checks on root."""
2097 at = self
2098 if g.unitTesting or not contents or not fileName or not fileName.endswith('.py'):
2099 return
2100 ok = True
2101 if at.checkPythonCodeOnWrite:
2102 ok = at.checkPythonSyntax(root, contents)
2103 if ok and at.runPyFlakesOnWrite:
2104 ok = self.runPyflakes(root)
2105 if not ok:
2106 g.app.syntax_error_files.append(g.shortFileName(fileName))
2107 #@+node:ekr.20090514111518.5663: *6* at.checkPythonSyntax
2108 def checkPythonSyntax(self, p, body):
2109 at = self
2110 try:
2111 body = body.replace('\r', '')
2112 fn = f"<node: {p.h}>"
2113 compile(body + '\n', fn, 'exec')
2114 return True
2115 except SyntaxError: # pragma: no cover
2116 if not g.unitTesting:
2117 at.syntaxError(p, body)
2118 except Exception: # pragma: no cover
2119 g.trace("unexpected exception")
2120 g.es_exception()
2121 return False
2122 #@+node:ekr.20090514111518.5666: *7* at.syntaxError (leoAtFile)
2123 def syntaxError(self, p, body): # pragma: no cover
2124 """Report a syntax error."""
2125 g.error(f"Syntax error in: {p.h}")
2126 typ, val, tb = sys.exc_info()
2127 message = hasattr(val, 'message') and val.message
2128 if message:
2129 g.es_print(message)
2130 if val is None:
2131 return
2132 lines = g.splitLines(body)
2133 n = val.lineno
2134 offset = val.offset or 0
2135 if n is None:
2136 return
2137 i = val.lineno - 1
2138 for j in range(max(0, i - 2), min(i + 2, len(lines) - 1)):
2139 line = lines[j].rstrip()
2140 if j == i:
2141 unl = p.get_UNL()
2142 g.es_print(f"{j+1:5}:* {line}", nodeLink=f"{unl}::-{j+1:d}") # Global line.
2143 g.es_print(' ' * (7 + offset) + '^')
2144 else:
2145 g.es_print(f"{j+1:5}: {line}")
2146 #@+node:ekr.20161021084954.1: *6* at.runPyflakes
2147 def runPyflakes(self, root): # pragma: no cover
2148 """Run pyflakes on the selected node."""
2149 try:
2150 from leo.commands import checkerCommands
2151 if checkerCommands.pyflakes:
2152 x = checkerCommands.PyflakesCommand(self.c)
2153 ok = x.run(root)
2154 return ok
2155 return True # Suppress error if pyflakes can not be imported.
2156 except Exception:
2157 g.es_exception()
2158 return True # Pretend all is well
2159 #@+node:ekr.20041005105605.198: *5* at.directiveKind4 (write logic)
2160 # These patterns exclude constructs such as @encoding.setter or @encoding(whatever)
2161 # However, they must allow @language python, @nocolor-node, etc.
2163 at_directive_kind_pattern = re.compile(r'\s*@([\w-]+)\s*')
2165 def directiveKind4(self, s, i):
2166 """
2167 Return the kind of at-directive or noDirective.
2169 Potential simplifications:
2170 - Using strings instead of constants.
2171 - Using additional regex's to recognize directives.
2172 """
2173 at = self
2174 n = len(s)
2175 if i >= n or s[i] != '@':
2176 j = g.skip_ws(s, i)
2177 if g.match_word(s, j, "@others"):
2178 return at.othersDirective
2179 if g.match_word(s, j, "@all"):
2180 return at.allDirective
2181 return at.noDirective
2182 table = (
2183 ("@all", at.allDirective),
2184 ("@c", at.cDirective),
2185 ("@code", at.codeDirective),
2186 ("@doc", at.docDirective),
2187 ("@others", at.othersDirective),
2188 ("@verbatim", at.startVerbatim))
2189 # ("@end_raw", at.endRawDirective), # #2276.
2190 # ("@raw", at.rawDirective), # #2276
2191 # Rewritten 6/8/2005.
2192 if i + 1 >= n or s[i + 1] in (' ', '\t', '\n'):
2193 # Bare '@' not recognized in cweb mode.
2194 return at.noDirective if at.language == "cweb" else at.atDirective
2195 if not s[i + 1].isalpha():
2196 return at.noDirective # Bug fix: do NOT return miscDirective here!
2197 if at.language == "cweb" and g.match_word(s, i, '@c'):
2198 return at.noDirective
2199 # When the language is elixir, @doc followed by a space and string delimiter
2200 # needs to be treated as plain text; the following does not enforce the
2201 # 'string delimiter' part of that. An @doc followed by something other than
2202 # a space will fall through to usual Leo @doc processing.
2203 if at.language == "elixir" and g.match_word(s, i, '@doc '): # pragma: no cover
2204 return at.noDirective
2205 for name, directive in table:
2206 if g.match_word(s, i, name):
2207 return directive
2208 # Support for add_directives plugin.
2209 # Use regex to properly distinguish between Leo directives
2210 # and python decorators.
2211 s2 = s[i:]
2212 m = self.at_directive_kind_pattern.match(s2)
2213 if m:
2214 word = m.group(1)
2215 if word not in g.globalDirectiveList:
2216 return at.noDirective
2217 s3 = s2[m.end(1) :]
2218 if s3 and s3[0] in ".(":
2219 return at.noDirective
2220 return at.miscDirective
2221 # An unusual case.
2222 return at.noDirective # pragma: no cover
2223 #@+node:ekr.20041005105605.200: *5* at.isSectionName
2224 # returns (flag, end). end is the index of the character after the section name.
2226 def isSectionName(self, s, i): # pragma: no cover
2228 at = self
2229 # Allow leading periods.
2230 while i < len(s) and s[i] == '.':
2231 i += 1
2232 if not g.match(s, i, at.section_delim1):
2233 return False, -1
2234 i = g.find_on_line(s, i, at.section_delim2)
2235 if i > -1:
2236 return True, i + len(at.section_delim2)
2237 return False, -1
2238 #@+node:ekr.20190111112442.1: *5* at.isWritable
2239 def isWritable(self, path): # pragma: no cover
2240 """Return True if the path is writable."""
2241 try:
2242 # os.access() may not exist on all platforms.
2243 ok = os.access(path, os.W_OK)
2244 except AttributeError:
2245 return True
2246 if not ok:
2247 g.es('read only:', repr(path), color='red')
2248 return ok
2249 #@+node:ekr.20041005105605.201: *5* at.os and allies
2250 #@+node:ekr.20041005105605.202: *6* at.oblank, oblanks & otabs
2251 def oblank(self):
2252 self.os(' ')
2254 def oblanks(self, n): # pragma: no cover
2255 self.os(' ' * abs(n))
2257 def otabs(self, n): # pragma: no cover
2258 self.os('\t' * abs(n))
2259 #@+node:ekr.20041005105605.203: *6* at.onl & onl_sent
2260 def onl(self):
2261 """Write a newline to the output stream."""
2262 self.os('\n') # **not** self.output_newline
2264 def onl_sent(self):
2265 """Write a newline to the output stream, provided we are outputting sentinels."""
2266 if self.sentinels:
2267 self.onl()
2268 #@+node:ekr.20041005105605.204: *6* at.os
2269 def os(self, s):
2270 """
2271 Append a string to at.outputList.
2273 All output produced by leoAtFile module goes here.
2274 """
2275 at = self
2276 if s.startswith(self.underindentEscapeString): # pragma: no cover
2277 try:
2278 junk, s = at.parseUnderindentTag(s)
2279 except Exception:
2280 at.exception("exception writing:" + s)
2281 return
2282 s = g.toUnicode(s, at.encoding)
2283 at.outputList.append(s)
2284 #@+node:ekr.20041005105605.205: *5* at.outputStringWithLineEndings
2285 def outputStringWithLineEndings(self, s): # pragma: no cover
2286 """
2287 Write the string s as-is except that we replace '\n' with the proper line ending.
2289 Calling self.onl() runs afoul of queued newlines.
2290 """
2291 at = self
2292 s = g.toUnicode(s, at.encoding)
2293 s = s.replace('\n', at.output_newline)
2294 self.os(s)
2295 #@+node:ekr.20190111045822.1: *5* at.precheck (calls shouldPrompt...)
2296 def precheck(self, fileName, root): # pragma: no cover
2297 """
2298 Check whether a dirty, potentially dangerous, file should be written.
2300 Return True if so. Return False *and* issue a warning otherwise.
2301 """
2302 at = self
2303 #
2304 # #1450: First, check that the directory exists.
2305 theDir = g.os_path_dirname(fileName)
2306 if theDir and not g.os_path_exists(theDir):
2307 at.error(f"Directory not found:\n{theDir}")
2308 return False
2309 #
2310 # Now check the file.
2311 if not at.shouldPromptForDangerousWrite(fileName, root):
2312 # Fix bug 889175: Remember the full fileName.
2313 at.rememberReadPath(fileName, root)
2314 return True
2315 #
2316 # Prompt if the write would overwrite the existing file.
2317 ok = self.promptForDangerousWrite(fileName)
2318 if ok:
2319 # Fix bug 889175: Remember the full fileName.
2320 at.rememberReadPath(fileName, root)
2321 return True
2322 #
2323 # Fix #1031: do not add @ignore here!
2324 g.es("not written:", fileName)
2325 return False
2326 #@+node:ekr.20050506090446.1: *5* at.putAtFirstLines
2327 def putAtFirstLines(self, s):
2328 """
2329 Write any @firstlines from string s.
2330 These lines are converted to @verbatim lines,
2331 so the read logic simply ignores lines preceding the @+leo sentinel.
2332 """
2333 at = self
2334 tag = "@first"
2335 i = 0
2336 while g.match(s, i, tag):
2337 i += len(tag)
2338 i = g.skip_ws(s, i)
2339 j = i
2340 i = g.skip_to_end_of_line(s, i)
2341 # Write @first line, whether empty or not
2342 line = s[j:i]
2343 at.os(line)
2344 at.onl()
2345 i = g.skip_nl(s, i)
2346 #@+node:ekr.20050506090955: *5* at.putAtLastLines
2347 def putAtLastLines(self, s):
2348 """
2349 Write any @last lines from string s.
2350 These lines are converted to @verbatim lines,
2351 so the read logic simply ignores lines following the @-leo sentinel.
2352 """
2353 at = self
2354 tag = "@last"
2355 # Use g.splitLines to preserve trailing newlines.
2356 lines = g.splitLines(s)
2357 n = len(lines)
2358 j = k = n - 1
2359 # Scan backwards for @last directives.
2360 while j >= 0:
2361 line = lines[j]
2362 if g.match(line, 0, tag):
2363 j -= 1
2364 elif not line.strip():
2365 j -= 1
2366 else:
2367 break # pragma: no cover (coverage bug)
2368 # Write the @last lines.
2369 for line in lines[j + 1 : k + 1]:
2370 if g.match(line, 0, tag):
2371 i = len(tag)
2372 i = g.skip_ws(line, i)
2373 at.os(line[i:])
2374 #@+node:ekr.20041005105605.206: *5* at.putDirective & helper
2375 def putDirective(self, s, i, p):
2376 r"""
2377 Output a sentinel a directive or reference s.
2379 It is important for PHP and other situations that \@first and \@last
2380 directives get translated to verbatim lines that do *not* include what
2381 follows the @first & @last directives.
2382 """
2383 at = self
2384 k = i
2385 j = g.skip_to_end_of_line(s, i)
2386 directive = s[i:j]
2387 if g.match_word(s, k, "@delims"):
2388 at.putDelims(directive, s, k)
2389 elif g.match_word(s, k, "@language"):
2390 self.putSentinel("@" + directive)
2391 elif g.match_word(s, k, "@comment"):
2392 self.putSentinel("@" + directive)
2393 elif g.match_word(s, k, "@last"):
2394 # #1307.
2395 if p.isAtCleanNode(): # pragma: no cover
2396 at.error(f"ignoring @last directive in {p.h!r}")
2397 g.es_print('@last is not valid in @clean nodes')
2398 # #1297.
2399 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2400 self.putSentinel("@@last")
2401 # Convert to an verbatim line _without_ anything else.
2402 else:
2403 at.error(f"ignoring @last directive in {p.h!r}") # pragma: no cover
2404 elif g.match_word(s, k, "@first"):
2405 # #1307.
2406 if p.isAtCleanNode(): # pragma: no cover
2407 at.error(f"ignoring @first directive in {p.h!r}")
2408 g.es_print('@first is not valid in @clean nodes')
2409 # #1297.
2410 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2411 self.putSentinel("@@first")
2412 # Convert to an verbatim line _without_ anything else.
2413 else:
2414 at.error(f"ignoring @first directive in {p.h!r}") # pragma: no cover
2415 else:
2416 self.putSentinel("@" + directive)
2417 i = g.skip_line(s, k)
2418 return i
2419 #@+node:ekr.20041005105605.207: *6* at.putDelims
2420 def putDelims(self, directive, s, k):
2421 """Put an @delims directive."""
2422 at = self
2423 # Put a space to protect the last delim.
2424 at.putSentinel(directive + " ") # 10/23/02: put @delims, not @@delims
2425 # Skip the keyword and whitespace.
2426 j = i = g.skip_ws(s, k + len("@delims"))
2427 # Get the first delim.
2428 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2429 i += 1
2430 if j < i:
2431 at.startSentinelComment = s[j:i]
2432 # Get the optional second delim.
2433 j = i = g.skip_ws(s, i)
2434 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2435 i += 1
2436 at.endSentinelComment = s[j:i] if j < i else ""
2437 else:
2438 at.writeError("Bad @delims directive") # pragma: no cover
2439 #@+node:ekr.20041005105605.210: *5* at.putIndent
2440 def putIndent(self, n, s=''): # pragma: no cover
2441 """Put tabs and spaces corresponding to n spaces,
2442 assuming that we are at the start of a line.
2444 Remove extra blanks if the line starts with the underindentEscapeString"""
2445 tag = self.underindentEscapeString
2446 if s.startswith(tag):
2447 n2, s2 = self.parseUnderindentTag(s)
2448 if n2 >= n:
2449 return
2450 if n > 0:
2451 n -= n2
2452 else:
2453 n += n2
2454 if n > 0:
2455 w = self.tab_width
2456 if w > 1:
2457 q, r = divmod(n, w)
2458 self.otabs(q)
2459 self.oblanks(r)
2460 else:
2461 self.oblanks(n)
2462 #@+node:ekr.20041005105605.211: *5* at.putInitialComment
2463 def putInitialComment(self): # pragma: no cover
2464 c = self.c
2465 s2 = c.config.output_initial_comment
2466 if s2:
2467 lines = s2.split("\\n")
2468 for line in lines:
2469 line = line.replace("@date", time.asctime())
2470 if line:
2471 self.putSentinel("@comment " + line)
2472 #@+node:ekr.20190111172114.1: *5* at.replaceFile & helpers
2473 def replaceFile(self, contents, encoding, fileName, root, ignoreBlankLines=False):
2474 """
2475 Write or create the given file from the contents.
2476 Return True if the original file was changed.
2477 """
2478 at, c = self, self.c
2479 if root:
2480 root.clearDirty()
2481 #
2482 # Create the timestamp (only for messages).
2483 if c.config.getBool('log-show-save-time', default=False): # pragma: no cover
2484 format = c.config.getString('log-timestamp-format') or "%H:%M:%S"
2485 timestamp = time.strftime(format) + ' '
2486 else:
2487 timestamp = ''
2488 #
2489 # Adjust the contents.
2490 assert isinstance(contents, str), g.callers()
2491 if at.output_newline != '\n': # pragma: no cover
2492 contents = contents.replace('\r', '').replace('\n', at.output_newline)
2493 #
2494 # If file does not exist, create it from the contents.
2495 fileName = g.os_path_realpath(fileName)
2496 sfn = g.shortFileName(fileName)
2497 if not g.os_path_exists(fileName):
2498 ok = g.writeFile(contents, encoding, fileName)
2499 if ok:
2500 c.setFileTimeStamp(fileName)
2501 if not g.unitTesting:
2502 g.es(f"{timestamp}created: {fileName}") # pragma: no cover
2503 if root:
2504 # Fix bug 889175: Remember the full fileName.
2505 at.rememberReadPath(fileName, root)
2506 at.checkPythonCode(contents, fileName, root)
2507 else:
2508 at.addToOrphanList(root) # pragma: no cover
2509 # No original file to change. Return value tested by a unit test.
2510 return False # No change to original file.
2511 #
2512 # Compare the old and new contents.
2513 old_contents = g.readFileIntoUnicodeString(fileName,
2514 encoding=at.encoding, silent=True)
2515 if not old_contents:
2516 old_contents = ''
2517 unchanged = (
2518 contents == old_contents
2519 or (not at.explicitLineEnding and at.compareIgnoringLineEndings(old_contents, contents))
2520 or ignoreBlankLines and at.compareIgnoringBlankLines(old_contents, contents))
2521 if unchanged:
2522 at.unchangedFiles += 1
2523 if not g.unitTesting and c.config.getBool(
2524 'report-unchanged-files', default=True):
2525 g.es(f"{timestamp}unchanged: {sfn}") # pragma: no cover
2526 # Leo 5.6: Check unchanged files.
2527 at.checkPyflakes(contents, fileName, root)
2528 return False # No change to original file.
2529 #
2530 # Warn if we are only adjusting the line endings.
2531 if at.explicitLineEnding: # pragma: no cover
2532 ok = (
2533 at.compareIgnoringLineEndings(old_contents, contents) or
2534 ignoreBlankLines and at.compareIgnoringLineEndings(
2535 old_contents, contents))
2536 if not ok:
2537 g.warning("correcting line endings in:", fileName)
2538 #
2539 # Write a changed file.
2540 ok = g.writeFile(contents, encoding, fileName)
2541 if ok:
2542 c.setFileTimeStamp(fileName)
2543 if not g.unitTesting:
2544 g.es(f"{timestamp}wrote: {sfn}") # pragma: no cover
2545 else: # pragma: no cover
2546 g.error('error writing', sfn)
2547 g.es('not written:', sfn)
2548 at.addToOrphanList(root)
2549 at.checkPythonCode(contents, fileName, root)
2550 # Check *after* writing the file.
2551 return ok
2552 #@+node:ekr.20190114061452.27: *6* at.compareIgnoringBlankLines
2553 def compareIgnoringBlankLines(self, s1, s2): # pragma: no cover
2554 """Compare two strings, ignoring blank lines."""
2555 assert isinstance(s1, str), g.callers()
2556 assert isinstance(s2, str), g.callers()
2557 if s1 == s2:
2558 return True
2559 s1 = g.removeBlankLines(s1)
2560 s2 = g.removeBlankLines(s2)
2561 return s1 == s2
2562 #@+node:ekr.20190114061452.28: *6* at.compareIgnoringLineEndings
2563 def compareIgnoringLineEndings(self, s1, s2): # pragma: no cover
2564 """Compare two strings, ignoring line endings."""
2565 assert isinstance(s1, str), (repr(s1), g.callers())
2566 assert isinstance(s2, str), (repr(s2), g.callers())
2567 if s1 == s2:
2568 return True
2569 # Wrong: equivalent to ignoreBlankLines!
2570 # s1 = s1.replace('\n','').replace('\r','')
2571 # s2 = s2.replace('\n','').replace('\r','')
2572 s1 = s1.replace('\r', '')
2573 s2 = s2.replace('\r', '')
2574 return s1 == s2
2575 #@+node:ekr.20211029052041.1: *5* at.scanRootForSectionDelims
2576 def scanRootForSectionDelims(self, root):
2577 """
2578 Scan root.b for an "@section-delims" directive.
2579 Set section_delim1 and section_delim2 ivars.
2580 """
2581 at = self
2582 # Set defaults.
2583 at.section_delim1 = '<<'
2584 at.section_delim2 = '>>'
2585 # Scan root.b.
2586 lines = []
2587 for s in g.splitLines(root.b):
2588 m = g.g_section_delims_pat.match(s)
2589 if m:
2590 lines.append(s)
2591 at.section_delim1 = m.group(1)
2592 at.section_delim2 = m.group(2)
2593 # Disallow multiple directives.
2594 if len(lines) > 1: # pragma: no cover
2595 at.error(f"Multiple @section-delims directives in {root.h}")
2596 g.es_print('using default delims')
2597 at.section_delim1 = '<<'
2598 at.section_delim2 = '>>'
2599 #@+node:ekr.20090514111518.5665: *5* at.tabNannyNode
2600 def tabNannyNode(self, p, body):
2601 try:
2602 readline = g.ReadLinesClass(body).next
2603 tabnanny.process_tokens(tokenize.generate_tokens(readline))
2604 except IndentationError: # pragma: no cover
2605 if g.unitTesting:
2606 raise
2607 junk2, msg, junk = sys.exc_info()
2608 g.error("IndentationError in", p.h)
2609 g.es('', str(msg))
2610 except tokenize.TokenError: # pragma: no cover
2611 if g.unitTesting:
2612 raise
2613 junk3, msg, junk = sys.exc_info()
2614 g.error("TokenError in", p.h)
2615 g.es('', str(msg))
2616 except tabnanny.NannyNag: # pragma: no cover
2617 if g.unitTesting:
2618 raise
2619 junk4, nag, junk = sys.exc_info()
2620 badline = nag.get_lineno()
2621 line = nag.get_line()
2622 message = nag.get_msg()
2623 g.error("indentation error in", p.h, "line", badline)
2624 g.es(message)
2625 line2 = repr(str(line))[1:-1]
2626 g.es("offending line:\n", line2)
2627 except Exception: # pragma: no cover
2628 g.trace("unexpected exception")
2629 g.es_exception()
2630 raise
2631 #@+node:ekr.20041005105605.216: *5* at.warnAboutOrpanAndIgnoredNodes
2632 # Called from putFile.
2634 def warnAboutOrphandAndIgnoredNodes(self): # pragma: no cover
2635 # Always warn, even when language=="cweb"
2636 at, root = self, self.root
2637 if at.errors:
2638 return # No need to repeat this.
2639 for p in root.self_and_subtree(copy=False):
2640 if not p.v.isVisited():
2641 at.writeError("Orphan node: " + p.h)
2642 if p.hasParent():
2643 g.blue("parent node:", p.parent().h)
2644 p = root.copy()
2645 after = p.nodeAfterTree()
2646 while p and p != after:
2647 if p.isAtAllNode():
2648 p.moveToNodeAfterTree()
2649 else:
2650 # #1050: test orphan bit.
2651 if p.isOrphan():
2652 at.writeError("Orphan node: " + p.h)
2653 if p.hasParent():
2654 g.blue("parent node:", p.parent().h)
2655 p.moveToThreadNext()
2656 #@+node:ekr.20041005105605.217: *5* at.writeError
2657 def writeError(self, message): # pragma: no cover
2658 """Issue an error while writing an @<file> node."""
2659 at = self
2660 if at.errors == 0:
2661 fn = at.targetFileName or 'unnamed file'
2662 g.es_error(f"errors writing: {fn}")
2663 at.error(message)
2664 at.addToOrphanList(at.root)
2665 #@+node:ekr.20041005105605.218: *5* at.writeException
2666 def writeException(self, fileName, root): # pragma: no cover
2667 at = self
2668 g.error("exception writing:", fileName)
2669 g.es_exception()
2670 if getattr(at, 'outputFile', None):
2671 at.outputFile.flush()
2672 at.outputFile.close()
2673 at.outputFile = None
2674 at.remove(fileName)
2675 at.addToOrphanList(root)
2676 #@+node:ekr.20041005105605.219: *3* at.Utilites
2677 #@+node:ekr.20041005105605.220: *4* at.error & printError
2678 def error(self, *args): # pragma: no cover
2679 at = self
2680 at.printError(*args)
2681 at.errors += 1
2683 def printError(self, *args): # pragma: no cover
2684 """Print an error message that may contain non-ascii characters."""
2685 at = self
2686 if at.errors:
2687 g.error(*args)
2688 else:
2689 g.warning(*args)
2690 #@+node:ekr.20041005105605.221: *4* at.exception
2691 def exception(self, message): # pragma: no cover
2692 self.error(message)
2693 g.es_exception()
2694 #@+node:ekr.20050104131929: *4* at.file operations...
2695 # Error checking versions of corresponding functions in Python's os module.
2696 #@+node:ekr.20050104131820: *5* at.chmod
2697 def chmod(self, fileName, mode): # pragma: no cover
2698 # Do _not_ call self.error here.
2699 if mode is None:
2700 return
2701 try:
2702 os.chmod(fileName, mode)
2703 except Exception:
2704 g.es("exception in os.chmod", fileName)
2705 g.es_exception()
2707 #@+node:ekr.20050104132018: *5* at.remove
2708 def remove(self, fileName): # pragma: no cover
2709 if not fileName:
2710 g.trace('No file name', g.callers())
2711 return False
2712 try:
2713 os.remove(fileName)
2714 return True
2715 except Exception:
2716 if not g.unitTesting:
2717 self.error(f"exception removing: {fileName}")
2718 g.es_exception()
2719 return False
2720 #@+node:ekr.20050104132026: *5* at.stat
2721 def stat(self, fileName): # pragma: no cover
2722 """Return the access mode of named file, removing any setuid, setgid, and sticky bits."""
2723 # Do _not_ call self.error here.
2724 try:
2725 mode = (os.stat(fileName))[0] & (7 * 8 * 8 + 7 * 8 + 7) # 0777
2726 except Exception:
2727 mode = None
2728 return mode
2730 #@+node:ekr.20090530055015.6023: *4* at.get/setPathUa
2731 def getPathUa(self, p):
2732 if hasattr(p.v, 'tempAttributes'):
2733 d = p.v.tempAttributes.get('read-path', {})
2734 return d.get('path')
2735 return ''
2737 def setPathUa(self, p, path):
2738 if not hasattr(p.v, 'tempAttributes'):
2739 p.v.tempAttributes = {}
2740 d = p.v.tempAttributes.get('read-path', {})
2741 d['path'] = path
2742 p.v.tempAttributes['read-path'] = d
2743 #@+node:ekr.20081216090156.4: *4* at.parseUnderindentTag
2744 # Important: this is part of the *write* logic.
2745 # It is called from at.os and at.putIndent.
2747 def parseUnderindentTag(self, s): # pragma: no cover
2748 tag = self.underindentEscapeString
2749 s2 = s[len(tag) :]
2750 # To be valid, the escape must be followed by at least one digit.
2751 i = 0
2752 while i < len(s2) and s2[i].isdigit():
2753 i += 1
2754 if i > 0:
2755 n = int(s2[:i])
2756 # Bug fix: 2012/06/05: remove any period following the count.
2757 # This is a new convention.
2758 if i < len(s2) and s2[i] == '.':
2759 i += 1
2760 return n, s2[i:]
2761 return 0, s
2762 #@+node:ekr.20090712050729.6017: *4* at.promptForDangerousWrite
2763 def promptForDangerousWrite(self, fileName, message=None): # pragma: no cover
2764 """Raise a dialog asking the user whether to overwrite an existing file."""
2765 at, c, root = self, self.c, self.root
2766 if at.cancelFlag:
2767 assert at.canCancelFlag
2768 return False
2769 if at.yesToAll:
2770 assert at.canCancelFlag
2771 return True
2772 if root and root.h.startswith('@auto-rst'):
2773 # Fix bug 50: body text lost switching @file to @auto-rst
2774 # Refuse to convert any @<file> node to @auto-rst.
2775 d = root.v.at_read if hasattr(root.v, 'at_read') else {}
2776 aList = sorted(d.get(fileName, []))
2777 for h in aList:
2778 if not h.startswith('@auto-rst'):
2779 g.es('can not convert @file to @auto-rst!', color='red')
2780 g.es('reverting to:', h)
2781 root.h = h
2782 c.redraw()
2783 return False
2784 if message is None:
2785 message = (
2786 f"{g.splitLongFileName(fileName)}\n"
2787 f"{g.tr('already exists.')}\n"
2788 f"{g.tr('Overwrite this file?')}")
2789 result = g.app.gui.runAskYesNoCancelDialog(c,
2790 title='Overwrite existing file?',
2791 yesToAllMessage="Yes To &All",
2792 message=message,
2793 cancelMessage="&Cancel (No To All)",
2794 )
2795 if at.canCancelFlag:
2796 # We are in the writeAll logic so these flags can be set.
2797 if result == 'cancel':
2798 at.cancelFlag = True
2799 elif result == 'yes-to-all':
2800 at.yesToAll = True
2801 return result in ('yes', 'yes-to-all')
2802 #@+node:ekr.20120112084820.10001: *4* at.rememberReadPath
2803 def rememberReadPath(self, fn, p):
2804 """
2805 Remember the files that have been read *and*
2806 the full headline (@<file> type) that caused the read.
2807 """
2808 v = p.v
2809 # Fix bug #50: body text lost switching @file to @auto-rst
2810 if not hasattr(v, 'at_read'):
2811 v.at_read = {} # pragma: no cover
2812 d = v.at_read
2813 aSet = d.get(fn, set())
2814 aSet.add(p.h)
2815 d[fn] = aSet
2816 #@+node:ekr.20080923070954.4: *4* at.scanAllDirectives
2817 def scanAllDirectives(self, p):
2818 """
2819 Scan p and p's ancestors looking for directives,
2820 setting corresponding AtFile ivars.
2821 """
2822 at, c = self, self.c
2823 d = c.scanAllDirectives(p)
2824 #
2825 # Language & delims: Tricky.
2826 lang_dict = d.get('lang-dict') or {}
2827 delims, language = None, None
2828 if lang_dict:
2829 # There was an @delims or @language directive.
2830 language = lang_dict.get('language')
2831 delims = lang_dict.get('delims')
2832 if not language:
2833 # No language directive. Look for @<file> nodes.
2834 # Do *not* used.get('language')!
2835 language = g.getLanguageFromAncestorAtFileNode(p) or 'python'
2836 at.language = language
2837 if not delims:
2838 delims = g.set_delims_from_language(language)
2839 #
2840 # Previously, setting delims was sometimes skipped, depending on kwargs.
2841 #@+<< Set comment strings from delims >>
2842 #@+node:ekr.20080923070954.13: *5* << Set comment strings from delims >> (at.scanAllDirectives)
2843 delim1, delim2, delim3 = delims
2844 # Use single-line comments if we have a choice.
2845 # delim1,delim2,delim3 now correspond to line,start,end
2846 if delim1:
2847 at.startSentinelComment = delim1
2848 at.endSentinelComment = "" # Must not be None.
2849 elif delim2 and delim3:
2850 at.startSentinelComment = delim2
2851 at.endSentinelComment = delim3
2852 else: # pragma: no cover
2853 #
2854 # Emergency!
2855 #
2856 # Issue an error only if at.language has been set.
2857 # This suppresses a message from the markdown importer.
2858 if not g.unitTesting and at.language:
2859 g.trace(repr(at.language), g.callers())
2860 g.es_print("unknown language: using Python comment delimiters")
2861 g.es_print("c.target_language:", c.target_language)
2862 at.startSentinelComment = "#" # This should never happen!
2863 at.endSentinelComment = ""
2864 #@-<< Set comment strings from delims >>
2865 #
2866 # Easy cases
2867 at.encoding = d.get('encoding') or c.config.default_derived_file_encoding
2868 lineending = d.get('lineending')
2869 at.explicitLineEnding = bool(lineending)
2870 at.output_newline = lineending or g.getOutputNewline(c=c)
2871 at.page_width = d.get('pagewidth') or c.page_width
2872 at.tab_width = d.get('tabwidth') or c.tab_width
2873 return {
2874 "encoding": at.encoding,
2875 "language": at.language,
2876 "lineending": at.output_newline,
2877 "pagewidth": at.page_width,
2878 "path": d.get('path'),
2879 "tabwidth": at.tab_width,
2880 }
2881 #@+node:ekr.20120110174009.9965: *4* at.shouldPromptForDangerousWrite
2882 def shouldPromptForDangerousWrite(self, fn, p): # pragma: no cover
2883 """
2884 Return True if Leo should warn the user that p is an @<file> node that
2885 was not read during startup. Writing that file might cause data loss.
2887 See #50: https://github.com/leo-editor/leo-editor/issues/50
2888 """
2889 trace = 'save' in g.app.debug
2890 sfn = g.shortFileName(fn)
2891 c = self.c
2892 efc = g.app.externalFilesController
2893 if p.isAtNoSentFileNode():
2894 # #1450.
2895 # No danger of overwriting a file.
2896 # It was never read.
2897 return False
2898 if not g.os_path_exists(fn):
2899 # No danger of overwriting fn.
2900 if trace:
2901 g.trace('Return False: does not exist:', sfn)
2902 return False
2903 # #1347: Prompt if the external file is newer.
2904 if efc:
2905 # Like c.checkFileTimeStamp.
2906 if c.sqlite_connection and c.mFileName == fn:
2907 # sqlite database file is never actually overwriten by Leo,
2908 # so do *not* check its timestamp.
2909 pass
2910 elif efc.has_changed(fn):
2911 if trace:
2912 g.trace('Return True: changed:', sfn)
2913 return True
2914 if hasattr(p.v, 'at_read'):
2915 # Fix bug #50: body text lost switching @file to @auto-rst
2916 d = p.v.at_read
2917 for k in d:
2918 # Fix bug # #1469: make sure k still exists.
2919 if (
2920 os.path.exists(k) and os.path.samefile(k, fn)
2921 and p.h in d.get(k, set())
2922 ):
2923 d[fn] = d[k]
2924 if trace:
2925 g.trace('Return False: in p.v.at_read:', sfn)
2926 return False
2927 aSet = d.get(fn, set())
2928 if trace:
2929 g.trace(f"Return {p.h not in aSet()}: p.h not in aSet(): {sfn}")
2930 return p.h not in aSet
2931 if trace:
2932 g.trace('Return True: never read:', sfn)
2933 return True # The file was never read.
2934 #@+node:ekr.20041005105605.20: *4* at.warnOnReadOnlyFile
2935 def warnOnReadOnlyFile(self, fn):
2936 # os.access() may not exist on all platforms.
2937 try:
2938 read_only = not os.access(fn, os.W_OK)
2939 except AttributeError: # pragma: no cover
2940 read_only = False
2941 if read_only:
2942 g.error("read only:", fn) # pragma: no cover
2943 #@-others
2944atFile = AtFile # compatibility
2945#@+node:ekr.20180602102448.1: ** class FastAtRead
2946class FastAtRead:
2947 """
2948 Read an exteral file, created from an @file tree.
2949 This is Vitalije's code, edited by EKR.
2950 """
2952 #@+others
2953 #@+node:ekr.20211030193146.1: *3* fast_at.__init__
2954 def __init__(self, c, gnx2vnode):
2956 self.c = c
2957 assert gnx2vnode is not None
2958 self.gnx2vnode = gnx2vnode # The global fc.gnxDict. Keys are gnx's, values are vnodes.
2959 self.path = None
2960 self.root = None
2961 # compiled patterns...
2962 self.after_pat = None
2963 self.all_pat = None
2964 self.code_pat = None
2965 self.comment_pat = None
2966 self.delims_pat = None
2967 self.doc_pat = None
2968 self.first_pat = None
2969 self.last_pat = None
2970 self.node_start_pat = None
2971 self.others_pat = None
2972 self.ref_pat = None
2973 self.section_delims_pat = None
2974 #@+node:ekr.20180602103135.3: *3* fast_at.get_patterns
2975 #@@nobeautify
2977 def get_patterns(self, comment_delims):
2978 """Create regex patterns for the given comment delims."""
2979 # This must be a function, because of @comments & @delims.
2980 comment_delim_start, comment_delim_end = comment_delims
2981 delim1 = re.escape(comment_delim_start)
2982 delim2 = re.escape(comment_delim_end or '')
2983 ref = g.angleBrackets(r'(.*)')
2984 table = (
2985 # These patterns must be mutually exclusive.
2986 ('after', fr'^\s*{delim1}@afterref{delim2}$'), # @afterref
2987 ('all', fr'^(\s*){delim1}@(\+|-)all\b(.*){delim2}$'), # @all
2988 ('code', fr'^\s*{delim1}@@c(ode)?{delim2}$'), # @c and @code
2989 ('comment', fr'^\s*{delim1}@@comment(.*){delim2}'), # @comment
2990 ('delims', fr'^\s*{delim1}@delims(.*){delim2}'), # @delims
2991 ('doc', fr'^\s*{delim1}@\+(at|doc)?(\s.*?)?{delim2}\n'), # @doc or @
2992 ('first', fr'^\s*{delim1}@@first{delim2}$'), # @first
2993 ('last', fr'^\s*{delim1}@@last{delim2}$'), # @last
2994 # @node
2995 ('node_start', fr'^(\s*){delim1}@\+node:([^:]+): \*(\d+)?(\*?) (.*){delim2}$'),
2996 ('others', fr'^(\s*){delim1}@(\+|-)others\b(.*){delim2}$'), # @others
2997 ('ref', fr'^(\s*){delim1}@(\+|-){ref}\s*{delim2}$'), # section ref
2998 # @section-delims
2999 ('section_delims', fr'^\s*{delim1}@@section-delims[ \t]+([^ \w\n\t]+)[ \t]+([^ \w\n\t]+)[ \t]*{delim2}$'),
3000 )
3001 # Set the ivars.
3002 for (name, pattern) in table:
3003 ivar = f"{name}_pat"
3004 assert hasattr(self, ivar), ivar
3005 setattr(self, ivar, re.compile(pattern))
3006 #@+node:ekr.20180602103135.2: *3* fast_at.scan_header
3007 header_pattern = re.compile(
3008 r'''
3009 ^(.+)@\+leo
3010 (-ver=(\d+))?
3011 (-thin)?
3012 (-encoding=(.*)(\.))?
3013 (.*)$''',
3014 re.VERBOSE,
3015 )
3017 def scan_header(self, lines):
3018 """
3019 Scan for the header line, which follows any @first lines.
3020 Return (delims, first_lines, i+1) or None
3021 """
3022 first_lines: List[str] = []
3023 i = 0 # To keep some versions of pylint happy.
3024 for i, line in enumerate(lines):
3025 m = self.header_pattern.match(line)
3026 if m:
3027 delims = m.group(1), m.group(8) or ''
3028 return delims, first_lines, i + 1
3029 first_lines.append(line)
3030 return None # pragma: no cover (defensive)
3031 #@+node:ekr.20180602103135.8: *3* fast_at.scan_lines
3032 def scan_lines(self, comment_delims, first_lines, lines, path, start):
3033 """Scan all lines of the file, creating vnodes."""
3034 #@+<< init scan_lines >>
3035 #@+node:ekr.20180602103135.9: *4* << init scan_lines >>
3036 #
3037 # Simple vars...
3038 afterref = False # True: the next line follows @afterref.
3039 clone_v = None # The root of the clone tree.
3040 comment_delim1, comment_delim2 = comment_delims # The start/end *comment* delims.
3041 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n') # To handle doc parts.
3042 first_i = 0 # Index into first array.
3043 in_doc = False # True: in @doc parts.
3044 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>' # True: cweb hack in effect.
3045 indent = 0 # The current indentation.
3046 level_stack = [] # Entries are (vnode, in_clone_tree)
3047 n_last_lines = 0 # The number of @@last directives seen.
3048 root_gnx_adjusted = False # True: suppress final checks.
3049 # #1065 so reads will not create spurious child nodes.
3050 root_seen = False # False: The next +@node sentinel denotes the root, regardless of gnx.
3051 section_delim1 = '<<'
3052 section_delim2 = '>>'
3053 section_reference_seen = False
3054 sentinel = comment_delim1 + '@' # Faster than a regex!
3055 # The stack is updated when at+others, at+<section>, or at+all is seen.
3056 stack = [] # Entries are (gnx, indent, body)
3057 # The spelling of at-verbatim sentinel
3058 verbatim_line = comment_delim1 + '@verbatim' + comment_delim2 + '\n'
3059 verbatim = False # True: the next line must be added without change.
3060 #
3061 # Init the parent vnode.
3062 #
3063 root_gnx = gnx = self.root.gnx
3064 context = self.c
3065 parent_v = self.root.v
3066 root_v = parent_v # Does not change.
3067 level_stack.append((root_v, False),)
3068 #
3069 # Init the gnx dict last.
3070 #
3071 gnx2vnode = self.gnx2vnode # Keys are gnx's, values are vnodes.
3072 gnx2body = {} # Keys are gnxs, values are list of body lines.
3073 gnx2vnode[gnx] = parent_v # Add gnx to the keys
3074 # Add gnx to the keys.
3075 # Body is the list of lines presently being accumulated.
3076 gnx2body[gnx] = body = first_lines
3077 #
3078 # Set the patterns
3079 self.get_patterns(comment_delims)
3080 #@-<< init scan_lines >>
3081 i = 0 # To keep pylint happy.
3082 for i, line in enumerate(lines[start:]):
3083 # Strip the line only once.
3084 strip_line = line.strip()
3085 if afterref:
3086 #@+<< handle afterref line>>
3087 #@+node:ekr.20211102052251.1: *4* << handle afterref line >>
3088 if body: # a List of lines.
3089 body[-1] = body[-1].rstrip() + line
3090 else:
3091 body = [line] # pragma: no cover
3092 afterref = False
3093 #@-<< handle afterref line>>
3094 continue
3095 if verbatim:
3096 #@+<< handle verbatim line >>
3097 #@+node:ekr.20211102052518.1: *4* << handle verbatim line >>
3098 # Previous line was verbatim *sentinel*. Append this line as it is.
3099 body.append(line)
3100 verbatim = False
3101 #@-<< handle verbatim line >>
3102 continue
3103 if line == verbatim_line: # <delim>@verbatim.
3104 verbatim = True
3105 continue
3106 #@+<< finalize line >>
3107 #@+node:ekr.20180602103135.10: *4* << finalize line >>
3108 # Undo the cweb hack.
3109 if is_cweb and line.startswith(sentinel):
3110 line = line[: len(sentinel)] + line[len(sentinel) :].replace('@@', '@')
3111 # Adjust indentation.
3112 if indent and line[:indent].isspace() and len(line) > indent:
3113 line = line[indent:]
3114 #@-<< finalize line >>
3115 if not in_doc and not strip_line.startswith(sentinel): # Faster than a regex!
3116 body.append(line)
3117 continue
3118 # These three sections might clear in_doc.
3119 #@+<< handle @others >>
3120 #@+node:ekr.20180602103135.14: *4* << handle @others >>
3121 m = self.others_pat.match(line)
3122 if m:
3123 in_doc = False
3124 if m.group(2) == '+': # opening sentinel
3125 body.append(f"{m.group(1)}@others{m.group(3) or ''}\n")
3126 stack.append((gnx, indent, body))
3127 indent += m.end(1) # adjust current identation
3128 else: # closing sentinel.
3129 # m.group(2) is '-' because the pattern matched.
3130 gnx, indent, body = stack.pop()
3131 continue
3132 #@-<< handle @others >>
3133 #@+<< handle section refs >>
3134 #@+node:ekr.20180602103135.18: *4* << handle section refs >>
3135 # Note: scan_header sets *comment* delims, not *section* delims.
3136 # This section coordinates with the section that handles @section-delims.
3137 m = self.ref_pat.match(line)
3138 if m:
3139 in_doc = False
3140 if m.group(2) == '+':
3141 # Any later @section-delims directive is a serious error.
3142 # This kind of error should have been caught by Leo's atFile write logic.
3143 section_reference_seen = True
3144 # open sentinel.
3145 body.append(m.group(1) + section_delim1 + m.group(3) + section_delim2 + '\n')
3146 stack.append((gnx, indent, body))
3147 indent += m.end(1)
3148 elif stack:
3149 # m.group(2) is '-' because the pattern matched.
3150 gnx, indent, body = stack.pop() # #1232: Only if the stack exists.
3151 continue # 2021/10/29: *always* continue.
3152 #@-<< handle section refs >>
3153 #@+<< handle node_start >>
3154 #@+node:ekr.20180602103135.19: *4* << handle node_start >>
3155 m = self.node_start_pat.match(line)
3156 if m:
3157 in_doc = False
3158 gnx, head = m.group(2), m.group(5)
3159 # m.group(3) is the level number, m.group(4) is the number of stars.
3160 level = int(m.group(3)) if m.group(3) else 1 + len(m.group(4))
3161 v = gnx2vnode.get(gnx)
3162 #
3163 # Case 1: The root @file node. Don't change the headline.
3164 if not root_seen and not v and not g.unitTesting:
3165 # Don't warn about a gnx mismatch in the root.
3166 root_gnx_adjusted = True # pragma: no cover
3167 if not root_seen:
3168 # Fix #1064: The node represents the root, regardless of the gnx!
3169 root_seen = True
3170 clone_v = None
3171 gnx2body[gnx] = body = []
3172 # This case can happen, but not in unit tests.
3173 if not v: # pragma: no cover
3174 # Fix #1064.
3175 v = root_v
3176 # This message is annoying when using git-diff.
3177 # if gnx != root_gnx:
3178 # g.es_print("using gnx from external file: %s" % (v.h), color='blue')
3179 gnx2vnode[gnx] = v
3180 v.fileIndex = gnx
3181 v.children = []
3182 continue
3183 #
3184 # Case 2: We are scanning the descendants of a clone.
3185 parent_v, clone_v = level_stack[level - 2]
3186 if v and clone_v:
3187 # The last version of the body and headline wins..
3188 gnx2body[gnx] = body = []
3189 v._headString = head
3190 # Update the level_stack.
3191 level_stack = level_stack[: level - 1]
3192 level_stack.append((v, clone_v),)
3193 # Always clear the children!
3194 v.children = []
3195 parent_v.children.append(v)
3196 continue
3197 #
3198 # Case 3: we are not already scanning the descendants of a clone.
3199 if v:
3200 # The *start* of a clone tree. Reset the children.
3201 clone_v = v
3202 v.children = []
3203 else:
3204 # Make a new vnode.
3205 v = leoNodes.VNode(context=context, gnx=gnx)
3206 #
3207 # The last version of the body and headline wins.
3208 gnx2vnode[gnx] = v
3209 gnx2body[gnx] = body = []
3210 v._headString = head
3211 #
3212 # Update the stack.
3213 level_stack = level_stack[: level - 1]
3214 level_stack.append((v, clone_v),)
3215 #
3216 # Update the links.
3217 assert v != root_v
3218 parent_v.children.append(v)
3219 v.parents.append(parent_v)
3220 continue
3221 #@-<< handle node_start >>
3222 if in_doc:
3223 #@+<< handle @c or @code >>
3224 #@+node:ekr.20211031033532.1: *4* << handle @c or @code >>
3225 # When delim_end exists the doc block:
3226 # - begins with the opening delim, alone on its own line
3227 # - ends with the closing delim, alone on its own line.
3228 # Both of these lines should be skipped.
3229 #
3230 # #1496: Retire the @doc convention.
3231 # An empty line is no longer a sentinel.
3232 if comment_delim2 and line in doc_skip:
3233 # doc_skip is (comment_delim1 + '\n', delim_end + '\n')
3234 continue
3235 #
3236 # Check for @c or @code.
3237 m = self.code_pat.match(line)
3238 if m:
3239 in_doc = False
3240 body.append('@code\n' if m.group(1) else '@c\n')
3241 continue
3242 #@-<< handle @c or @code >>
3243 else:
3244 #@+<< handle @ or @doc >>
3245 #@+node:ekr.20211031033754.1: *4* << handle @ or @doc >>
3246 m = self.doc_pat.match(line)
3247 if m:
3248 # @+at or @+doc?
3249 doc = '@doc' if m.group(1) == 'doc' else '@'
3250 doc2 = m.group(2) or '' # Trailing text.
3251 if doc2:
3252 body.append(f"{doc}{doc2}\n")
3253 else:
3254 body.append(doc + '\n')
3255 # Enter @doc mode.
3256 in_doc = True
3257 continue
3258 #@-<< handle @ or @doc >>
3259 if line.startswith(comment_delim1 + '@-leo'): # Faster than a regex!
3260 # The @-leo sentinel adds *nothing* to the text.
3261 i += 1
3262 break
3263 # Order doesn't matter.
3264 #@+<< handle @all >>
3265 #@+node:ekr.20180602103135.13: *4* << handle @all >>
3266 m = self.all_pat.match(line)
3267 if m:
3268 # @all tells Leo's *write* code not to check for undefined sections.
3269 # Here, in the read code, we merely need to add it to the body.
3270 # Pushing and popping the stack may not be necessary, but it can't hurt.
3271 if m.group(2) == '+': # opening sentinel
3272 body.append(f"{m.group(1)}@all{m.group(3) or ''}\n")
3273 stack.append((gnx, indent, body))
3274 else: # closing sentinel.
3275 # m.group(2) is '-' because the pattern matched.
3276 gnx, indent, body = stack.pop()
3277 gnx2body[gnx] = body
3278 continue
3279 #@-<< handle @all >>
3280 #@+<< handle afterref >>
3281 #@+node:ekr.20180603063102.1: *4* << handle afterref >>
3282 m = self.after_pat.match(line)
3283 if m:
3284 afterref = True
3285 continue
3286 #@-<< handle afterref >>
3287 #@+<< handle @first and @last >>
3288 #@+node:ekr.20180606053919.1: *4* << handle @first and @last >>
3289 m = self.first_pat.match(line)
3290 if m:
3291 # pylint: disable=no-else-continue
3292 if 0 <= first_i < len(first_lines):
3293 body.append('@first ' + first_lines[first_i])
3294 first_i += 1
3295 continue
3296 else: # pragma: no cover
3297 g.trace(f"\ntoo many @first lines: {path}")
3298 print('@first is valid only at the start of @<file> nodes\n')
3299 g.printObj(first_lines, tag='first_lines')
3300 g.printObj(lines[start : i + 2], tag='lines[start:i+2]')
3301 continue
3302 m = self.last_pat.match(line)
3303 if m:
3304 # Just increment the count of the expected last lines.
3305 # We'll fill in the @last line directives after we see the @-leo directive.
3306 n_last_lines += 1
3307 continue
3308 #@-<< handle @first and @last >>
3309 #@+<< handle @comment >>
3310 #@+node:ekr.20180621050901.1: *4* << handle @comment >>
3311 # http://leoeditor.com/directives.html#part-4-dangerous-directives
3312 m = self.comment_pat.match(line)
3313 if m:
3314 # <1, 2 or 3 comment delims>
3315 delims = m.group(1).strip()
3316 # Whatever happens, retain the @delims line.
3317 body.append(f"@comment {delims}\n")
3318 delim1, delim2, delim3 = g.set_delims_from_string(delims)
3319 # delim1 is always the single-line delimiter.
3320 if delim1:
3321 comment_delim1, comment_delim2 = delim1, ''
3322 else:
3323 comment_delim1, comment_delim2 = delim2, delim3
3324 #
3325 # Within these delimiters:
3326 # - double underscores represent a newline.
3327 # - underscores represent a significant space,
3328 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3329 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3330 # Recalculate all delim-related values
3331 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3332 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3333 sentinel = comment_delim1 + '@'
3334 #
3335 # Recalculate the patterns.
3336 comment_delims = comment_delim1, comment_delim2
3337 self.get_patterns(comment_delims)
3338 continue
3339 #@-<< handle @comment >>
3340 #@+<< handle @delims >>
3341 #@+node:ekr.20180608104836.1: *4* << handle @delims >>
3342 m = self.delims_pat.match(line)
3343 if m:
3344 # Get 1 or 2 comment delims
3345 # Whatever happens, retain the original @delims line.
3346 delims = m.group(1).strip()
3347 body.append(f"@delims {delims}\n")
3348 #
3349 # Parse the delims.
3350 self.delims_pat = re.compile(r'^([^ ]+)\s*([^ ]+)?')
3351 m2 = self.delims_pat.match(delims)
3352 if not m2: # pragma: no cover
3353 g.trace(f"Ignoring invalid @delims: {line!r}")
3354 continue
3355 comment_delim1 = m2.group(1)
3356 comment_delim2 = m2.group(2) or ''
3357 #
3358 # Within these delimiters:
3359 # - double underscores represent a newline.
3360 # - underscores represent a significant space,
3361 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3362 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3363 # Recalculate all delim-related values
3364 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3365 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3366 sentinel = comment_delim1 + '@'
3367 #
3368 # Recalculate the patterns
3369 comment_delims = comment_delim1, comment_delim2
3370 self.get_patterns(comment_delims)
3371 continue
3372 #@-<< handle @delims >>
3373 #@+<< handle @section-delims >>
3374 #@+node:ekr.20211030033211.1: *4* << handle @section-delims >>
3375 m = self.section_delims_pat.match(line)
3376 if m:
3377 if section_reference_seen: # pragma: no cover
3378 # This is a serious error.
3379 # This kind of error should have been caught by Leo's atFile write logic.
3380 g.es_print('section-delims seen after a section reference', color='red')
3381 else:
3382 # Carefully update the section reference pattern!
3383 section_delim1 = d1 = re.escape(m.group(1))
3384 section_delim2 = d2 = re.escape(m.group(2) or '')
3385 self.ref_pat = re.compile(fr'^(\s*){comment_delim1}@(\+|-){d1}(.*){d2}\s*{comment_delim2}$')
3386 body.append(f"@section-delims {m.group(1)} {m.group(2)}\n")
3387 continue
3388 #@-<< handle @section-delims >>
3389 # These sections must be last, in this order.
3390 #@+<< handle remaining @@ lines >>
3391 #@+node:ekr.20180603135602.1: *4* << handle remaining @@ lines >>
3392 # @first, @last, @delims and @comment generate @@ sentinels,
3393 # So this must follow all of those.
3394 if line.startswith(comment_delim1 + '@@'):
3395 ii = len(comment_delim1) + 1 # on second '@'
3396 jj = line.rfind(comment_delim2) if comment_delim2 else -1
3397 body.append(line[ii:jj] + '\n')
3398 continue
3399 #@-<< handle remaining @@ lines >>
3400 if in_doc:
3401 #@+<< handle remaining @doc lines >>
3402 #@+node:ekr.20180606054325.1: *4* << handle remaining @doc lines >>
3403 if comment_delim2:
3404 # doc lines are unchanged.
3405 body.append(line)
3406 continue
3407 # Doc lines start with start_delim + one blank.
3408 # #1496: Retire the @doc convention.
3409 # #2194: Strip lws.
3410 tail = line.lstrip()[len(comment_delim1) + 1 :]
3411 if tail.strip():
3412 body.append(tail)
3413 else:
3414 body.append('\n')
3415 continue
3416 #@-<< handle remaining @doc lines >>
3417 #@+<< handle remaining @ lines >>
3418 #@+node:ekr.20180602103135.17: *4* << handle remaining @ lines >>
3419 # Handle an apparent sentinel line.
3420 # This *can* happen after the git-diff or refresh-from-disk commands.
3421 #
3422 if 1: # pragma: no cover (defensive)
3423 # This assert verifies the short-circuit test.
3424 assert strip_line.startswith(sentinel), (repr(sentinel), repr(line))
3425 # A useful trace.
3426 g.trace(
3427 f"{g.shortFileName(self.path)}: "
3428 f"warning: inserting unexpected line: {line.rstrip()!r}"
3429 )
3430 # #2213: *Do* insert the line, with a warning.
3431 body.append(line)
3432 #@-<< handle remaining @ lines >>
3433 else:
3434 # No @-leo sentinel!
3435 return # pragma: no cover
3436 #@+<< final checks >>
3437 #@+node:ekr.20211104054823.1: *4* << final checks >>
3438 if g.unitTesting:
3439 # Unit tests must use the proper value for root.gnx.
3440 assert not root_gnx_adjusted
3441 assert not stack, stack
3442 assert root_gnx == gnx, (root_gnx, gnx)
3443 elif root_gnx_adjusted: # pragma: no cover
3444 pass # Don't check!
3445 elif stack: # pragma: no cover
3446 g.error('scan_lines: Stack should be empty')
3447 g.printObj(stack, tag='stack')
3448 elif root_gnx != gnx: # pragma: no cover
3449 g.error('scan_lines: gnx error')
3450 g.es_print(f"root_gnx: {root_gnx} != gnx: {gnx}")
3451 #@-<< final checks >>
3452 #@+<< insert @last lines >>
3453 #@+node:ekr.20211103101453.1: *4* << insert @last lines >>
3454 tail_lines = lines[start + i :]
3455 if tail_lines:
3456 # Convert the trailing lines to @last directives.
3457 last_lines = [f"@last {z.rstrip()}\n" for z in tail_lines]
3458 # Add the lines to the dictionary of lines.
3459 gnx2body[gnx] = gnx2body[gnx] + last_lines
3460 # Warn if there is an unexpected number of last lines.
3461 if n_last_lines != len(last_lines): # pragma: no cover
3462 n1 = n_last_lines
3463 n2 = len(last_lines)
3464 g.trace(f"Expected {n1} trailing line{g.plural(n1)}, got {n2}")
3465 #@-<< insert @last lines >>
3466 #@+<< post pass: set all body text>>
3467 #@+node:ekr.20211104054426.1: *4* << post pass: set all body text>>
3468 # Set the body text.
3469 assert root_v.gnx in gnx2vnode, root_v
3470 assert root_v.gnx in gnx2body, root_v
3471 for key in gnx2body:
3472 body = gnx2body.get(key)
3473 v = gnx2vnode.get(key)
3474 assert v, (key, v)
3475 v._bodyString = g.toUnicode(''.join(body))
3476 #@-<< post pass: set all body text>>
3477 #@+node:ekr.20180603170614.1: *3* fast_at.read_into_root
3478 def read_into_root(self, contents, path, root):
3479 """
3480 Parse the file's contents, creating a tree of vnodes
3481 anchored in root.v.
3482 """
3483 self.path = path
3484 self.root = root
3485 sfn = g.shortFileName(path)
3486 contents = contents.replace('\r', '')
3487 lines = g.splitLines(contents)
3488 data = self.scan_header(lines)
3489 if not data: # pragma: no cover
3490 g.trace(f"Invalid external file: {sfn}")
3491 return False
3492 # Clear all children.
3493 # Previously, this had been done in readOpenFile.
3494 root.v._deleteAllChildren()
3495 comment_delims, first_lines, start_i = data
3496 self.scan_lines(comment_delims, first_lines, lines, path, start_i)
3497 return True
3498 #@-others
3499#@-others
3500#@@language python
3501#@@tabwidth -4
3502#@@pagewidth 60
3504#@-leo