Coverage for C:\leo.repo\leo-editor\leo\core\leoShadow.py: 66%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20080708094444.1: * @file leoShadow.py
4#@@first
5#@+<< docstring >>
6#@+node:ekr.20080708094444.78: ** << docstring >>
7"""
8leoShadow.py
10This code allows users to use Leo with files which contain no sentinels
11and still have information flow in both directions between outlines and
12derived files.
14Private files contain sentinels: they live in the Leo-shadow subdirectory.
15Public files contain no sentinels: they live in the parent (main) directory.
17When Leo first reads an @shadow we create a file without sentinels in the regular directory.
19The slightly hard thing to do is to pick up changes from the file without
20sentinels, and put them into the file with sentinels.
22Settings:
23- @string shadow_subdir (default: .leo_shadow): name of the shadow directory.
25- @string shadow_prefix (default: x): prefix of shadow files.
26 This prefix allows the shadow file and the original file to have different names.
27 This is useful for name-based tools like py.test.
28"""
29#@-<< docstring >>
30#@+<< imports >>
31#@+node:ekr.20080708094444.52: ** << imports >> (leoShadow)
32import difflib
33import os
34import pprint
35from typing import List
36from leo.core import leoGlobals as g
37#@-<< imports >>
38#@+others
39#@+node:ekr.20080708094444.80: ** class ShadowController
40class ShadowController:
41 """A class to manage @shadow files"""
42 #@+others
43 #@+node:ekr.20080708094444.79: *3* x.ctor & x.reloadSettings
44 def __init__(self, c, trace=False, trace_writers=False):
45 """Ctor for ShadowController class."""
46 self.c = c
47 # Opcode dispatch dict.
48 self.dispatch_dict = {
49 'delete': self.op_delete,
50 'equal': self.op_equal,
51 'insert': self.op_insert,
52 'replace': self.op_replace,
53 }
54 # File encoding.
55 self.encoding = c.config.default_derived_file_encoding
56 # Configuration: set in reloadSettings.
57 self.shadow_subdir = None
58 self.shadow_prefix = None
59 self.shadow_in_home_dir = None
60 self.shadow_subdir = None
61 # Error handling...
62 self.errors = 0
63 self.last_error = '' # The last error message, regardless of whether it was actually shown.
64 self.trace = False
65 # Support for goto-line.
66 self.line_mapping = []
67 self.reloadSettings()
69 def reloadSettings(self):
70 """ShadowController.reloadSettings."""
71 c = self.c
72 self.shadow_subdir = c.config.getString('shadow-subdir') or '.leo_shadow'
73 self.shadow_prefix = c.config.getString('shadow-prefix') or ''
74 self.shadow_in_home_dir = c.config.getBool('shadow-in-home-dir', default=False)
75 self.shadow_subdir = g.os_path_normpath(self.shadow_subdir)
76 #@+node:ekr.20080711063656.1: *3* x.File utils
77 #@+node:ekr.20080711063656.7: *4* x.baseDirName
78 def baseDirName(self):
79 c = self.c
80 filename = c.fileName()
81 if filename:
82 return g.os_path_dirname(g.os_path_finalize(filename)) # 1341
83 print('')
84 self.error('Can not compute shadow path: .leo file has not been saved')
85 return None
86 #@+node:ekr.20080711063656.4: *4* x.dirName and pathName
87 def dirName(self, filename):
88 """Return the directory for filename."""
89 x = self
90 return g.os_path_dirname(x.pathName(filename))
92 def pathName(self, filename):
93 """Return the full path name of filename."""
94 x = self
95 theDir = x.baseDirName()
96 return theDir and g.os_path_finalize_join(theDir, filename) # 1341
97 #@+node:ekr.20080712080505.3: *4* x.isSignificantPublicFile
98 def isSignificantPublicFile(self, fn):
99 """
100 This tells the AtFile.read logic whether to import a public file
101 or use an existing public file.
102 """
103 return (
104 g.os_path_exists(fn) and
105 g.os_path_isfile(fn) and
106 g.os_path_getsize(fn) > 10
107 )
108 #@+node:ekr.20080710082231.19: *4* x.makeShadowDirectory
109 def makeShadowDirectory(self, fn):
110 """Make a shadow directory for the **public** fn."""
111 x = self
112 path = x.shadowDirName(fn)
113 if not g.os_path_exists(path):
114 # Force the creation of the directories.
115 g.makeAllNonExistentDirectories(path)
116 return g.os_path_exists(path) and g.os_path_isdir(path)
117 #@+node:ekr.20080713091247.1: *4* x.replaceFileWithString
118 def replaceFileWithString(self, encoding, fileName, s):
119 """
120 Replace the file with s if s is different from theFile's contents.
122 Return True if theFile was changed.
123 """
124 x, c = self, self.c
125 exists = g.os_path_exists(fileName)
126 if exists:
127 # Read the file. Return if it is the same.
128 s2, e = g.readFileIntoString(fileName)
129 if s2 is None:
130 return False
131 if s == s2:
132 report = c.config.getBool('report-unchanged-files', default=True)
133 if report and not g.unitTesting:
134 g.es('unchanged:', fileName)
135 return False
136 # Issue warning if directory does not exist.
137 theDir = g.os_path_dirname(fileName)
138 if theDir and not g.os_path_exists(theDir):
139 if not g.unitTesting:
140 x.error(f"not written: {fileName} directory not found")
141 return False
142 # Replace the file.
143 try:
144 with open(fileName, 'wb') as f:
145 # Fix bug 1243847: unicode error when saving @shadow nodes.
146 f.write(g.toEncodedString(s, encoding=encoding))
147 c.setFileTimeStamp(fileName)
148 # Fix #1053. This is an *ancient* bug.
149 if not g.unitTesting:
150 kind = 'wrote' if exists else 'created'
151 g.es(f"{kind:>6}: {fileName}")
152 return True
153 except IOError:
154 x.error(f"unexpected exception writing file: {fileName}")
155 g.es_exception()
156 return False
157 #@+node:ekr.20080711063656.6: *4* x.shadowDirName and shadowPathName
158 def shadowDirName(self, filename):
159 """Return the directory for the shadow file corresponding to filename."""
160 x = self
161 return g.os_path_dirname(x.shadowPathName(filename))
163 def shadowPathName(self, filename):
164 """Return the full path name of filename, resolved using c.fileName()"""
165 x = self
166 c = x.c
167 baseDir = x.baseDirName()
168 fileDir = g.os_path_dirname(filename)
169 # 2011/01/26: bogomil: redirect shadow dir
170 if self.shadow_in_home_dir:
171 # Each .leo file has a separate shadow_cache in base dir
172 fname = "_".join(
173 [os.path.splitext(os.path.basename(c.mFileName))[0], "shadow_cache"])
174 # On Windows incorporate the drive letter to the private file path
175 if os.name == "nt":
176 fileDir = fileDir.replace(':', '%')
177 # build the chache path as a subdir of the base dir
178 fileDir = "/".join([baseDir, fname, fileDir])
179 return baseDir and g.os_path_finalize_join( # 1341
180 baseDir,
181 fileDir, # Bug fix: honor any directories specified in filename.
182 x.shadow_subdir,
183 x.shadow_prefix + g.shortFileName(filename))
184 #@+node:ekr.20080708192807.1: *3* x.Propagation
185 #@+node:ekr.20080708094444.35: *4* x.check_output
186 def check_output(self):
187 """Check that we produced a valid output."""
188 x = self
189 lines1 = x.b
190 junk, sents1 = x.separate_sentinels(x.old_sent_lines, x.marker)
191 lines2, sents2 = x.separate_sentinels(x.results, x.marker)
192 ok = lines1 == lines2 and sents1 == sents2
193 if g.unitTesting:
194 # The unit test will report the error.
195 return ok
196 if lines1 != lines2:
197 g.trace()
198 d = difflib.Differ()
199 aList = list(d.compare(lines2, x.b))
200 pprint.pprint(aList)
201 if sents1 != sents2:
202 x.show_error(
203 lines1=sents1,
204 lines2=sents2,
205 message="Sentinals not preserved!",
206 lines1_message="old sentinels",
207 lines2_message="new sentinels")
208 return ok
209 #@+node:ekr.20080708094444.38: *4* x.propagate_changed_lines (main algorithm) & helpers
210 def propagate_changed_lines(
211 self, new_public_lines, old_private_lines, marker, p=None):
212 #@+<< docstring >>
213 #@+node:ekr.20150207044400.9: *5* << docstring >>
214 """
215 The Mulder update algorithm, revised by EKR.
217 Use the diff between the old and new public lines to insperse sentinels
218 from old_private_lines into the result.
220 The algorithm never deletes or rearranges sentinels. However, verbatim
221 sentinels may be inserted or deleted as needed.
222 """
223 #@-<< docstring >>
224 x = self
225 x.init_ivars(new_public_lines, old_private_lines, marker)
226 sm = difflib.SequenceMatcher(None, x.a, x.b)
227 # Ensure leading sentinels are put first.
228 x.put_sentinels(0)
229 x.sentinels[0] = []
230 for tag, ai, aj, bi, bj in sm.get_opcodes():
231 f = x.dispatch_dict.get(tag, x.op_bad)
232 f(tag, ai, aj, bi, bj)
233 # Put the trailing sentinels & check the result.
234 x.results.extend(x.trailing_sentinels)
235 # check_output is likely to be more buggy than the code under test.
236 # x.check_output()
237 return x.results
238 #@+node:ekr.20150207111757.180: *5* x.dump_args
239 def dump_args(self):
240 """Dump the argument lines."""
241 x = self
242 table = (
243 (x.old_sent_lines, 'old private lines'),
244 (x.a, 'old public lines'),
245 (x.b, 'new public lines'),
246 )
247 for lines, title in table:
248 x.dump_lines(lines, title)
249 g.pr()
250 #@+node:ekr.20150207111757.178: *5* x.dump_lines
251 def dump_lines(self, lines, title):
252 """Dump the given lines."""
253 print(f"\n{title}...\n")
254 for i, line in enumerate(lines):
255 g.pr(f"{i:4} {line!r}")
256 #@+node:ekr.20150209044257.6: *5* x.init_data
257 def init_data(self):
258 """
259 Init x.sentinels and x.trailing_sentinels arrays.
260 Return the list of non-sentinel lines in x.old_sent_lines.
261 """
262 x = self
263 lines = x.old_sent_lines
264 sentinels: List[str] = []
265 # The sentinels preceding each non-sentinel line,
266 # not including @verbatim sentinels.
267 new_lines = []
268 # A list of all non-sentinel lines found. Should match x.a.
269 x.sentinels = []
270 # A list of lists of sentinels preceding each line.
271 i = 0
272 while i < len(lines):
273 line = lines[i]
274 i += 1
275 if x.marker.isVerbatimSentinel(line):
276 # Do *not* include the @verbatim sentinel.
277 if i < len(lines):
278 line = lines[i]
279 i += 1
280 x.sentinels.append(sentinels)
281 sentinels = []
282 new_lines.append(line)
283 else:
284 x.verbatim_error()
285 elif x.marker.isSentinel(line):
286 sentinels.append(line)
287 else:
288 x.sentinels.append(sentinels)
289 sentinels = []
290 new_lines.append(line)
291 x.trailing_sentinels = sentinels
292 return new_lines
293 #@+node:ekr.20080708094444.40: *5* x.init_ivars
294 def init_ivars(self, new_public_lines, old_private_lines, marker):
295 """Init all ivars used by propagate_changed_lines & its helpers."""
296 x = self
297 x.delim1, x.delim2 = marker.getDelims()
298 x.marker = marker
299 x.old_sent_lines = old_private_lines
300 x.results = []
301 x.verbatim_line = f"{x.delim1}@verbatim{x.delim2}\n"
302 old_public_lines = x.init_data()
303 x.b = x.preprocess(new_public_lines)
304 x.a = x.preprocess(old_public_lines)
305 #@+node:ekr.20150207044400.16: *5* x.op_bad
306 def op_bad(self, tag, ai, aj, bi, bj):
307 """Report an unexpected opcode."""
308 x = self
309 x.error(f"unknown SequenceMatcher opcode: {tag!r}")
310 #@+node:ekr.20150207044400.12: *5* x.op_delete
311 def op_delete(self, tag, ai, aj, bi, bj):
312 """Handle the 'delete' opcode."""
313 x = self
314 for i in range(ai, aj):
315 x.put_sentinels(i)
316 #@+node:ekr.20150207044400.13: *5* x.op_equal
317 def op_equal(self, tag, ai, aj, bi, bj):
318 """Handle the 'equal' opcode."""
319 x = self
320 assert aj - ai == bj - bi and x.a[ai:aj] == x.b[bi:bj]
321 for i in range(ai, aj):
322 x.put_sentinels(i)
323 x.put_plain_line(x.a[i])
324 # works because x.lines[ai:aj] == x.lines[bi:bj]
325 #@+node:ekr.20150207044400.14: *5* x.op_insert
326 def op_insert(self, tag, ai, aj, bi, bj):
327 """Handle the 'insert' opcode."""
328 x = self
329 for i in range(bi, bj):
330 x.put_plain_line(x.b[i])
331 # Prefer to put sentinels after inserted nodes.
332 # Requires a call to x.put_sentinels(0) before the main loop.
333 #@+node:ekr.20150207044400.15: *5* x.op_replace
334 def op_replace(self, tag, ai, aj, bi, bj):
335 """Handle the 'replace' opcode."""
336 x = self
337 if 1:
338 # Intersperse sentinels and lines.
339 b_lines = x.b[bi:bj]
340 for i in range(ai, aj):
341 x.put_sentinels(i)
342 if b_lines:
343 x.put_plain_line(b_lines.pop(0))
344 # Put any trailing lines.
345 while b_lines:
346 x.put_plain_line(b_lines.pop(0))
347 else:
348 # Feasible. Causes the present unit tests to fail.
349 for i in range(ai, aj):
350 x.put_sentinels(i)
351 for i in range(bi, bj):
352 x.put_plain_line(x.b[i])
353 #@+node:ekr.20150208060128.7: *5* x.preprocess
354 def preprocess(self, lines):
355 """
356 Preprocess public lines, adding newlines as needed.
357 This happens before the diff.
358 """
359 result = []
360 for line in lines:
361 if not line.endswith('\n'):
362 line = line + '\n'
363 result.append(line)
364 return result
365 #@+node:ekr.20150208223018.4: *5* x.put_plain_line
366 def put_plain_line(self, line):
367 """Put a plain line to x.results, inserting verbatim lines if necessary."""
368 x = self
369 if x.marker.isSentinel(line):
370 x.results.append(x.verbatim_line)
371 if x.trace:
372 print(f"put {repr(x.verbatim_line)}")
373 x.results.append(line)
374 if x.trace:
375 print(f"put {line!r}")
376 #@+node:ekr.20150209044257.8: *5* x.put_sentinels
377 def put_sentinels(self, i):
378 """Put all the sentinels to the results"""
379 x = self
380 if 0 <= i < len(x.sentinels):
381 sentinels = x.sentinels[i]
382 if x.trace:
383 g.trace(f"{i:3} {sentinels}")
384 x.results.extend(sentinels)
385 #@+node:ekr.20080708094444.36: *4* x.propagate_changes
386 def propagate_changes(self, old_public_file, old_private_file):
387 """
388 Propagate the changes from the public file (without_sentinels)
389 to the private file (with_sentinels)
390 """
391 x, at = self, self.c.atFileCommands
392 at.errors = 0
393 self.encoding = at.encoding
394 s = at.readFileToUnicode(old_private_file) # Sets at.encoding and inits at.readLines.
395 old_private_lines = g.splitLines(s or '') # #1466.
396 s = at.readFileToUnicode(old_public_file)
397 if at.encoding != self.encoding:
398 g.trace(f"can not happen: encoding mismatch: {at.encoding} {self.encoding}")
399 at.encoding = self.encoding
400 old_public_lines = g.splitLines(s)
401 if 0:
402 g.trace(f"\nprivate lines...{old_private_file}")
403 for s in old_private_lines:
404 g.trace(type(s), isinstance(s, str), repr(s))
405 g.trace(f"\npublic lines...{old_public_file}")
406 for s in old_public_lines:
407 g.trace(type(s), isinstance(s, str), repr(s))
408 marker = x.markerFromFileLines(old_private_lines, old_private_file)
409 new_private_lines = x.propagate_changed_lines(
410 old_public_lines, old_private_lines, marker)
411 # Never create the private file here!
412 fn = old_private_file
413 exists = g.os_path_exists(fn)
414 different = new_private_lines != old_private_lines
415 copy = exists and different
416 # 2010/01/07: check at.errors also.
417 if copy and x.errors == 0 and at.errors == 0:
418 s = ''.join(new_private_lines)
419 x.replaceFileWithString(at.encoding, fn, s)
420 return copy
421 #@+node:bwmulder.20041231170726: *4* x.updatePublicAndPrivateFiles
422 def updatePublicAndPrivateFiles(self, root, fn, shadow_fn):
423 """handle crucial @shadow read logic.
425 This will be called only if the public and private files both exist."""
426 x = self
427 if x.isSignificantPublicFile(fn):
428 # Update the private shadow file from the public file.
429 written = x.propagate_changes(fn, shadow_fn)
430 if written:
431 x.message(f"updated private {shadow_fn} from public {fn}")
432 else:
433 pass
434 # Don't write *anything*.
435 # if 0: # This causes considerable problems.
436 # # Create the public file from the private shadow file.
437 # x.copy_file_removing_sentinels(shadow_fn,fn)
438 # x.message("created public %s from private %s " % (fn, shadow_fn))
439 #@+node:ekr.20080708094444.89: *3* x.Utils...
440 #@+node:ekr.20080708094444.85: *4* x.error & message & verbatim_error
441 def error(self, s, silent=False):
442 x = self
443 if not silent:
444 g.error(s)
445 # For unit testing.
446 x.last_error = s
447 x.errors += 1
449 def message(self, s):
450 g.es_print(s, color='orange')
452 def verbatim_error(self):
453 x = self
454 x.error('file syntax error: nothing follows verbatim sentinel')
455 g.trace(g.callers())
456 #@+node:ekr.20090529125512.6122: *4* x.markerFromFileLines & helper
457 def markerFromFileLines(self, lines, fn):
458 """Return the sentinel delimiter comment to be used for filename."""
459 at, x = self.c.atFileCommands, self
460 s = x.findLeoLine(lines)
461 ok, junk, start, end, junk = at.parseLeoSentinel(s)
462 if end:
463 delims = '', start, end
464 else:
465 delims = start, '', ''
466 return x.Marker(delims)
467 #@+node:ekr.20090529125512.6125: *5* x.findLeoLine
468 def findLeoLine(self, lines):
469 """Return the @+leo line, or ''."""
470 for line in lines:
471 i = line.find('@+leo')
472 if i != -1:
473 return line
474 return ''
475 #@+node:ekr.20080708094444.9: *4* x.markerFromFileName
476 def markerFromFileName(self, filename):
477 """Return the sentinel delimiter comment to be used for filename."""
478 x = self
479 if not filename:
480 return None
481 root, ext = g.os_path_splitext(filename)
482 if ext == '.tmp':
483 root, ext = os.path.splitext(root)
484 delims = g.comment_delims_from_extension(filename)
485 marker = x.Marker(delims)
486 return marker
487 #@+node:ekr.20080708094444.29: *4* x.separate_sentinels
488 def separate_sentinels(self, lines, marker):
489 """
490 Separates regular lines from sentinel lines.
491 Do not return @verbatim sentinels.
493 Returns (regular_lines, sentinel_lines)
494 """
495 x = self
496 regular_lines = []
497 sentinel_lines = []
498 i = 0
499 while i < len(lines):
500 line = lines[i]
501 if marker.isVerbatimSentinel(line):
502 # Add the plain line that *looks* like a sentinel,
503 # but *not* the preceding @verbatim sentinel itself.
504 # Adding the actual sentinel would spoil the sentinel test when
505 # the user adds or deletes a line requiring an @verbatim line.
506 i += 1
507 if i < len(lines):
508 line = lines[i]
509 regular_lines.append(line)
510 else:
511 x.verbatim_error()
512 elif marker.isSentinel(line):
513 sentinel_lines.append(line)
514 else:
515 regular_lines.append(line)
516 i += 1
517 return regular_lines, sentinel_lines
518 #@+node:ekr.20080708094444.33: *4* x.show_error & helper
519 def show_error(self, lines1, lines2, message, lines1_message, lines2_message):
520 x = self
521 banner1 = '=' * 30
522 banner2 = '-' * 30
523 g.es_print(f"{banner1}\n{message}\n{banner1}\n{lines1_message}\n{banner2}")
524 x.show_error_lines(lines1, 'shadow_errors.tmp1')
525 g.es_print(f"\n{banner1}\n{lines2_message}\n{banner1}")
526 x.show_error_lines(lines2, 'shadow_errors.tmp2')
527 g.es_print('\n@shadow did not pick up the external changes correctly')
528 # g.es_print('Please check shadow.tmp1 and shadow.tmp2 for differences')
529 #@+node:ekr.20080822065427.4: *5* x.show_error_lines
530 def show_error_lines(self, lines, fileName):
531 for i, line in enumerate(lines):
532 g.es_print(f"{i:3} {line!r}")
533 if False: # Only for major debugging.
534 try:
535 f1 = open(fileName, "w")
536 for s in lines:
537 f1.write(repr(s))
538 f1.close()
539 except IOError:
540 g.es_exception()
541 g.es_print('can not open', fileName)
542 #@+node:ekr.20090529061522.5727: *3* class x.Marker
543 class Marker:
544 """A class representing comment delims in @shadow files."""
545 #@+others
546 #@+node:ekr.20090529061522.6257: *4* ctor & repr
547 def __init__(self, delims):
548 """Ctor for Marker class."""
549 delim1, delim2, delim3 = delims
550 self.delim1 = delim1 # Single-line comment delim.
551 self.delim2 = delim2 # Block comment starting delim.
552 self.delim3 = delim3 # Block comment ending delim.
553 if not delim1 and not delim2:
554 # if g.unitTesting:
555 # assert False,repr(delims)
556 self.delim1 = g.app.language_delims_dict.get('unknown_language')
558 def __repr__(self):
559 if self.delim1:
560 delims = self.delim1
561 else:
562 delims = f"{self.delim2} {self.delim3}"
563 return f"<Marker: delims: {delims!r}>"
564 #@+node:ekr.20090529061522.6258: *4* getDelims
565 def getDelims(self):
566 """Return the pair of delims to be used in sentinel lines."""
567 if self.delim1:
568 return self.delim1, ''
569 return self.delim2, self.delim3
570 #@+node:ekr.20090529061522.6259: *4* isSentinel
571 def isSentinel(self, s, suffix=''):
572 """Return True is line s contains a valid sentinel comment."""
573 s = s.strip()
574 if self.delim1 and s.startswith(self.delim1):
575 return s.startswith(self.delim1 + '@' + suffix)
576 if self.delim2:
577 return s.startswith(
578 self.delim2 + '@' + suffix) and s.endswith(self.delim3)
579 return False
580 #@+node:ekr.20090529061522.6260: *4* isVerbatimSentinel
581 def isVerbatimSentinel(self, s):
582 """Return True if s is an @verbatim sentinel."""
583 return self.isSentinel(s, suffix='verbatim')
584 #@-others
585 #@-others
586#@-others
587#@@language python
588#@@tabwidth -4
589#@@pagewidth 70
590#@-leo