Coverage for src/lektor_ng/builder.py: 87%
675 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-09-05 10:32 +0000
1from __future__ import annotations
3import hashlib
4import os
5import shutil
6import sqlite3
7import stat
8import sys
9from collections import deque, namedtuple
10from collections.abc import Sized
11from contextlib import contextmanager
12from dataclasses import dataclass
13from itertools import chain
14from typing import IO, Any
16import click
18from lektor_ng.build_programs import builtin_build_programs
19from lektor_ng.buildfailures import FailureController
20from lektor_ng.constants import PRIMARY_ALT
21from lektor_ng.context import Context
22from lektor_ng.reporter import reporter
23from lektor_ng.sourcesearch import find_files
24from lektor_ng.utils import (
25 create_temp,
26 fs_enc,
27 process_extra_flags,
28 prune_file_and_folder,
29)
32def create_tables(con):
33 can_disable_rowid = (3, 8, 2) <= sqlite3.sqlite_version_info
34 if can_disable_rowid:
35 without_rowid = "without rowid"
36 else:
37 without_rowid = ""
39 try:
40 is_virtual_exists = con.execute(
41 """ select count(*) from pragma_table_info('artifacts')
42 where name='is_virtual';
43 """
44 ).fetchone()[0]
45 if not is_virtual_exists:
46 con.execute("""drop table if exists artifacts""")
47 con.execute(
48 f"""
49 create table if not exists artifacts (
50 artifact text,
51 source text,
52 source_mtime integer,
53 source_size integer,
54 source_checksum text,
55 is_dir integer,
56 is_virtual integer,
57 is_primary_source integer,
58 primary key (artifact, source)
59 ) {without_rowid};
60 """
61 )
62 con.execute(
63 """
64 create index if not exists artifacts_source on artifacts (
65 source
66 );
67 """
68 )
69 con.execute(
70 f"""
71 create table if not exists artifact_config_hashes (
72 artifact text,
73 config_hash text,
74 primary key (artifact)
75 ) {without_rowid};
76 """
77 )
78 con.execute(
79 f"""
80 create table if not exists dirty_sources (
81 source text,
82 primary key (source)
83 ) {without_rowid};
84 """
85 )
86 con.execute(
87 f"""
88 create table if not exists source_info (
89 path text,
90 alt text,
91 lang text,
92 type text,
93 source text,
94 title text,
95 primary key (path, alt, lang)
96 ) {without_rowid};
97 """
98 )
99 finally:
100 con.close()
103def _placeholders(values: Sized) -> str:
104 """Return SQL '?' placeholders for an array or set of values."""
105 return ",".join(["?"] * len(values))
108class BuildState:
109 def __init__(self, builder, path_cache):
110 self.builder = builder
112 self.updated_artifacts = []
113 self.failed_artifacts = []
114 self.path_cache = path_cache
116 @property
117 def pad(self):
118 """The pad for this buildstate."""
119 return self.builder.pad
121 @property
122 def env(self):
123 """The environment backing this buildstate."""
124 return self.builder.env
126 @property
127 def config(self):
128 """The config for this buildstate."""
129 return self.builder.pad.db.config
131 def notify_failure(self, artifact, exc_info):
132 """Notify about a failure. This marks a failed artifact and stores
133 a failure.
134 """
135 self.failed_artifacts.append(artifact)
136 self.builder.failure_controller.store_failure(artifact.artifact_name, exc_info)
137 reporter.report_failure(artifact, exc_info)
139 def get_file_info(self, filename):
140 if filename:
141 return self.path_cache.get_file_info(filename)
142 return None
144 def to_source_filename(self, filename):
145 return self.path_cache.to_source_filename(filename)
147 def get_virtual_source_info(self, virtual_source_path, alt=None):
148 virtual_source = self.pad.get(virtual_source_path, alt=alt)
149 if virtual_source is not None:
150 mtime = virtual_source.get_mtime(self.path_cache)
151 checksum = virtual_source.get_checksum(self.path_cache)
152 else:
153 mtime = checksum = None
154 return VirtualSourceInfo(virtual_source_path, alt, mtime, checksum)
156 def connect_to_database(self):
157 """Returns a database connection for the build state db."""
158 return self.builder.connect_to_database()
160 def get_destination_filename(self, artifact_name):
161 """Returns the destination filename for an artifact name."""
162 return os.path.join(
163 self.builder.destination_path,
164 artifact_name.strip("/").replace("/", os.path.sep),
165 )
167 def artifact_name_from_destination_filename(self, filename):
168 """Returns the artifact name for a destination filename."""
169 dst = self.builder.destination_path
170 filename = os.path.join(dst, filename)
171 if filename.startswith(dst):
172 filename = filename[len(dst) :].lstrip(os.path.sep)
173 if os.path.altsep:
174 filename = filename.lstrip(os.path.altsep)
175 return filename.replace(os.path.sep, "/")
177 def new_artifact(self, artifact_name, sources=None, source_obj=None, extra=None, config_hash=None):
178 """Creates a new artifact and returns it."""
179 dst_filename = self.get_destination_filename(artifact_name)
180 key = self.artifact_name_from_destination_filename(dst_filename)
181 return Artifact(
182 self,
183 key,
184 dst_filename,
185 sources,
186 source_obj=source_obj,
187 extra=extra,
188 config_hash=config_hash,
189 )
191 def artifact_exists(self, artifact_name):
192 """Given an artifact name this checks if it was already produced."""
193 dst_filename = self.get_destination_filename(artifact_name)
194 return os.path.exists(dst_filename)
196 def get_artifact_dependency_infos(self, artifact_name, sources):
197 con = self.connect_to_database()
198 try:
199 cur = con.cursor()
200 rv = list(self._iter_artifact_dependency_infos(cur, artifact_name, sources))
201 finally:
202 con.close()
203 return rv
205 def _iter_artifact_dependency_infos(self, cur, artifact_name, sources):
206 """This iterates over all dependencies as file info objects."""
207 cur.execute(
208 """
209 select source, source_mtime, source_size,
210 source_checksum, is_dir, is_virtual
211 from artifacts
212 where artifact = ?
213 """,
214 [artifact_name],
215 )
216 rv = cur.fetchall()
218 found = set()
219 for path, mtime, size, checksum, is_dir, is_virtual in rv:
220 if is_virtual:
221 assert "@" in path
222 vpath, alt = _unpack_virtual_source_path(path)
223 yield path, VirtualSourceInfo(vpath, alt, mtime, checksum)
224 else:
225 file_info = FileInfo(self.env, path, mtime, size, checksum, bool(is_dir))
226 filename = self.to_source_filename(file_info.filename)
227 found.add(filename)
228 yield filename, file_info
230 # In any case we also iterate over our direct sources, even if the
231 # build state does not know about them yet. This can be caused by
232 # an initial build or a change in original configuration.
233 for source in sources:
234 filename = self.to_source_filename(source)
235 if filename not in found:
236 yield source, None
238 def write_source_info(self, info):
239 """Writes the source info into the database. The source info is
240 an instance of :class:`lektor.build_programs.SourceInfo`.
241 """
242 reporter.report_write_source_info(info)
243 source = self.to_source_filename(info.filename)
244 con = self.connect_to_database()
245 try:
246 cur = con.cursor()
247 for lang, title in info.title_i18n.items():
248 cur.execute(
249 """
250 insert or replace into source_info
251 (path, alt, lang, type, source, title)
252 values (?, ?, ?, ?, ?, ?)
253 """,
254 [info.path, info.alt, lang, info.type, source, title],
255 )
256 con.commit()
257 finally:
258 con.close()
260 def prune_source_infos(self):
261 """Remove all source infos of files that no longer exist."""
262 MAX_VARS = 999 # Default SQLITE_MAX_VARIABLE_NUMBER.
263 con = self.connect_to_database()
264 to_clean = []
265 try:
266 cur = con.cursor()
267 cur.execute(
268 """
269 select distinct source from source_info
270 """
271 )
272 for (source,) in cur.fetchall():
273 fs_path = os.path.join(self.env.root_path, source)
274 if not os.path.exists(fs_path):
275 to_clean.append(source)
277 if to_clean:
278 for i in range(0, len(to_clean), MAX_VARS):
279 chunk = to_clean[i : i + MAX_VARS]
280 cur.execute(
281 f"""
282 delete from source_info
283 where source in ({_placeholders(chunk)})
284 """,
285 chunk,
286 )
288 con.commit()
289 finally:
290 con.close()
292 for source in to_clean:
293 reporter.report_prune_source_info(source)
295 def remove_artifact(self, artifact_name):
296 """Removes an artifact from the build state."""
297 con = self.connect_to_database()
298 try:
299 cur = con.cursor()
300 cur.execute(
301 """
302 delete from artifacts where artifact = ?
303 """,
304 [artifact_name],
305 )
306 con.commit()
307 finally:
308 con.close()
310 def _any_sources_are_dirty(self, cur, sources):
311 """Given a list of sources this checks if any of them are marked
312 as dirty.
313 """
314 sources = [self.to_source_filename(x) for x in sources]
315 if not sources:
316 return False
318 cur.execute(
319 f"""
320 select source from dirty_sources
321 where source in ({_placeholders(sources)})
322 limit 1
323 """,
324 sources,
325 )
326 return cur.fetchone() is not None
328 @staticmethod
329 def _get_artifact_config_hash(cur, artifact_name):
330 """Returns the artifact's config hash."""
331 cur.execute(
332 """
333 select config_hash from artifact_config_hashes
334 where artifact = ?
335 """,
336 [artifact_name],
337 )
338 rv = cur.fetchone()
339 return rv[0] if rv else None
341 def check_artifact_is_current(self, artifact_name, sources, config_hash):
342 con = self.connect_to_database()
343 cur = con.cursor()
344 try:
345 # The artifact config changed
346 if config_hash != self._get_artifact_config_hash(cur, artifact_name):
347 return False
349 # If one of our source files is explicitly marked as dirty in the
350 # build state, we are not current.
351 if self._any_sources_are_dirty(cur, sources):
352 return False
354 # If we do have an already existing artifact, we need to check if
355 # any of the source files we depend on changed.
356 for _, info in self._iter_artifact_dependency_infos(cur, artifact_name, sources):
357 # if we get a missing source info it means that we never
358 # saw this before. This means we need to build it.
359 if info is None:
360 return False
362 if info.is_changed(self):
363 return False
365 return True
366 finally:
367 con.close()
369 def iter_existing_artifacts(self):
370 """Scan output directory for artifacts.
372 Returns an iterable of the artifact_names for artifacts found.
373 """
374 is_ignored = self.env.is_ignored_artifact
376 def _unignored(filenames):
377 return filter(lambda fn: not is_ignored(fn), filenames)
379 dst = self.builder.destination_path
380 for dirpath, dirnames, filenames in os.walk(dst):
381 dirnames[:] = _unignored(dirnames)
382 for filename in _unignored(filenames):
383 full_path = os.path.join(dst, dirpath, filename)
384 yield self.artifact_name_from_destination_filename(full_path)
386 def iter_unreferenced_artifacts(self, all=False):
387 """Finds all unreferenced artifacts in the build folder and yields
388 them.
389 """
390 if all:
391 yield from self.iter_existing_artifacts()
393 con = self.connect_to_database()
394 cur = con.cursor()
396 def _is_unreferenced(artifact_name):
397 # Check whether any of the primary sources for the artifact
398 # exist and — if the source can be resolved to a record —
399 # correspond to non-hidden records.
400 cur.execute(
401 """
402 SELECT DISTINCT source, path, alt
403 FROM artifacts LEFT JOIN source_info USING(source)
404 WHERE artifact = ?
405 AND is_primary_source""",
406 [artifact_name],
407 )
408 for source, path, alt in cur.fetchall():
409 if self.get_file_info(source).exists:
410 if path is None:
411 return False # no record to check
412 record = self.pad.get(path, alt)
413 if record is None:
414 # I'm not sure this should happen, but be safe
415 return False
416 if record.is_visible:
417 return False
418 # no sources exist, or those that do belong to hidden records
419 return True
421 try:
422 yield from filter(_is_unreferenced, self.iter_existing_artifacts())
423 finally:
424 con.close()
426 def iter_artifacts(self):
427 """Iterates over all artifact and their file infos.."""
428 con = self.connect_to_database()
429 try:
430 cur = con.cursor()
431 cur.execute(
432 """
433 select distinct artifact from artifacts order by artifact
434 """
435 )
436 rows = cur.fetchall()
437 con.close()
438 for (artifact_name,) in rows:
439 path = self.get_destination_filename(artifact_name)
440 info = FileInfo(self.builder.env, path)
441 if info.exists:
442 yield artifact_name, info
443 finally:
444 con.close()
446 def vacuum(self):
447 """Vacuums the build db."""
448 con = self.connect_to_database()
449 try:
450 con.execute("vacuum")
451 finally:
452 con.close()
455def _describe_fs_path_for_checksum(path):
456 """Given a file system path this returns a basic description of what
457 this is. This is used for checksum hashing on directories.
458 """
459 # This is not entirely correct as it does not detect changes for
460 # contents from alternatives. However for the moment it's good
461 # enough.
462 if os.path.isfile(path):
463 return b"\x01"
464 if os.path.isfile(os.path.join(path, "contents.lr")):
465 return b"\x02"
466 if os.path.isdir(path):
467 return b"\x03"
468 return b"\x00"
471class _ArtifactSourceInfo:
472 """Base for classes that contain freshness data about artifact sources.
474 Concrete subclasses include FileInfo and VirtualSourceInfo.
475 """
477 def is_changed(self, build_state: BuildState) -> bool:
478 """Determine whether source has changed."""
479 raise NotImplementedError()
482class FileInfo(_ArtifactSourceInfo):
483 """A file info object holds metainformation of a file so that changes
484 can be detected easily.
485 """
487 def __init__(self, env, filename, mtime=None, size=None, checksum=None, is_dir=None):
488 self.env = env
489 self.filename = filename
490 if mtime is not None and size is not None and is_dir is not None:
491 self._stat = (mtime, size, is_dir)
492 else:
493 self._stat = None
494 self._checksum = checksum
496 def _get_stat(self):
497 rv = self._stat
498 if rv is not None:
499 return rv
501 try:
502 st = os.stat(self.filename)
503 mtime = int(st.st_mtime)
504 if stat.S_ISDIR(st.st_mode):
505 size = len(os.listdir(self.filename))
506 is_dir = True
507 else:
508 size = int(st.st_size)
509 is_dir = False
510 rv = mtime, size, is_dir
511 except OSError:
512 rv = 0, -1, False
513 self._stat = rv
514 return rv
516 @property
517 def mtime(self):
518 """The timestamp of the last modification."""
519 return self._get_stat()[0]
521 @property
522 def size(self):
523 """The size of the file in bytes. If the file is actually a
524 dictionary then the size is actually the number of files in it.
525 """
526 return self._get_stat()[1]
528 @property
529 def is_dir(self):
530 """Is this a directory?"""
531 return self._get_stat()[2]
533 @property
534 def exists(self):
535 return self.size >= 0
537 @property
538 def checksum(self):
539 """The checksum of the file or directory."""
540 rv = self._checksum
541 if rv is not None:
542 return rv
544 try:
545 h = hashlib.sha1()
546 if os.path.isdir(self.filename):
547 h.update(b"DIR\x00")
548 for filename in sorted(os.listdir(self.filename)):
549 if self.env.is_uninteresting_source_name(filename):
550 continue
551 if isinstance(filename, str):
552 filename = filename.encode("utf-8")
553 h.update(filename)
554 h.update(_describe_fs_path_for_checksum(os.path.join(self.filename, filename.decode("utf-8"))))
555 h.update(b"\x00")
556 else:
557 with open(self.filename, "rb") as f:
558 while 1:
559 chunk = f.read(16 * 1024)
560 if not chunk:
561 break
562 h.update(chunk)
563 checksum = h.hexdigest()
564 except OSError:
565 checksum = "0" * 40
566 self._checksum = checksum
567 return checksum
569 @property
570 def filename_and_checksum(self):
571 """Like 'filename:checksum'."""
572 return f"{self.filename}:{self.checksum}"
574 def unchanged(self, other):
575 """Given another file info checks if the are similar enough to
576 not consider it changed.
577 """
578 if not isinstance(other, FileInfo):
579 raise TypeError(f"'other' must be a FileInfo, not {other!r}")
581 if self.mtime != other.mtime or self.size != other.size:
582 return False
583 # If mtime and size match, we skip the checksum comparison which
584 # might require a file read which we do not want in those cases.
585 # (Except if it's a directory, then we won't do that)
586 if not self.is_dir:
587 return True
588 return self.checksum == other.checksum
590 def is_changed(self, build_state: BuildState) -> bool:
591 other = build_state.get_file_info(self.filename)
592 return not self.unchanged(other)
595def _pack_virtual_source_path(path, alt):
596 """Pack VirtualSourceObject's path and alt into a single string.
598 The full identity key for a VirtualSourceObject is its ``path`` along with its
599 ``alt``. (Two VirtualSourceObjects with differing alts are not the same object.)
601 This functions packs the (path, alt) pair into a single string for storage
602 in the ``artifacts.path`` of the buildstate database.
604 Note that if alternatives are not configured for the current site, there is
605 only one alt, so we safely omit the alt from the packed path.
607 """
608 if alt is None or alt == PRIMARY_ALT:
609 return path
610 return f"{alt}@{path}"
613def _unpack_virtual_source_path(packed):
614 """Unpack VirtualSourceObject's path and alt from packed path.
616 This is the inverse of _pack_virtual_source_path.
617 """
618 alt, sep, path = packed.partition("@")
619 if not sep:
620 raise ValueError("A packed virtual source path must include at least one '@'")
621 if "@" not in path:
622 path, alt = packed, None
623 return path, alt
626@dataclass
627class VirtualSourceInfo(_ArtifactSourceInfo):
628 path: str
629 alt: str | None
630 mtime: int | None = None
631 checksum: str | None = None
633 def unchanged(self, other):
634 if not isinstance(other, VirtualSourceInfo):
635 raise TypeError(f"'other' must be a VirtualSourceInfo, not {other!r}")
637 if (self.path, self.alt) != (other.path, other.alt):
638 raise ValueError(f"trying to compare mismatched virtual paths: {self!r}.unchanged({other!r})")
640 return (self.mtime, self.checksum) == (other.mtime, other.checksum)
642 def is_changed(self, build_state: BuildState) -> bool:
643 other = build_state.get_virtual_source_info(self.path, self.alt)
644 return not self.unchanged(other)
647artifacts_row = namedtuple(
648 "artifacts_row",
649 [
650 "artifact",
651 "source",
652 "source_mtime",
653 "source_size",
654 "source_checksum",
655 "is_dir",
656 "is_virtual",
657 "is_primary_source",
658 ],
659)
662class Artifact:
663 """This class represents a build artifact."""
665 def __init__(
666 self,
667 build_state,
668 artifact_name,
669 dst_filename,
670 sources,
671 *,
672 source_obj=None,
673 extra=None,
674 config_hash=None,
675 ):
676 self.build_state = build_state
677 self.artifact_name = artifact_name
678 self.dst_filename = dst_filename
679 self.sources = sources
680 self.in_update_block = False
681 self.updated = False
682 self.source_obj = source_obj
683 self.extra = extra
684 self.config_hash = config_hash
686 self._new_artifact_file = None
687 self._pending_update_ops = []
689 def __repr__(self):
690 return f"<{self.__class__.__name__} {self.dst_filename!r}>"
692 @property
693 def is_current(self):
694 """Checks if the artifact is current."""
695 # If the artifact does not exist, we're not current.
696 if not os.path.isfile(self.dst_filename):
697 return False
699 return self.build_state.check_artifact_is_current(self.artifact_name, self.sources, self.config_hash)
701 def get_dependency_infos(self):
702 return self.build_state.get_artifact_dependency_infos(self.artifact_name, self.sources)
704 def ensure_dir(self):
705 """Creates the directory if it does not exist yet."""
706 dst_dir = os.path.dirname(self.dst_filename)
707 try:
708 os.makedirs(dst_dir)
709 except OSError:
710 pass
712 def open(self, mode: str = "rb", encoding: str | None = None, ensure_dir: bool = True) -> IO[Any]:
713 """Opens the artifact for reading or writing. This is transaction
714 safe by writing into a temporary file and by moving it over the
715 actual source in commit.
716 """
717 if self._new_artifact_file is not None:
718 return open(self._new_artifact_file, mode, encoding=encoding)
720 if "r" in mode:
721 return open(self.dst_filename, mode, encoding=encoding)
723 if ensure_dir:
724 self.ensure_dir()
726 fd, self._new_artifact_file = create_temp(
727 prefix=".__trans",
728 dir=os.path.dirname(self.dst_filename),
729 text="b" not in mode,
730 )
731 return open(fd, mode, encoding=encoding)
733 def replace_with_file(self, filename, ensure_dir=True, copy=False):
734 """This is similar to open but it will move over a given named
735 file. The file will be deleted by a rollback or renamed by a
736 commit.
737 """
738 if ensure_dir:
739 self.ensure_dir()
740 if copy:
741 with self.open("wb") as df, open(filename, "rb") as sf:
742 shutil.copyfileobj(sf, df)
743 else:
744 self._new_artifact_file = filename
746 def render_template_into(self, template_name, this, **extra):
747 """Renders a template into the artifact."""
748 rv = self.build_state.env.render_template(template_name, self.build_state.pad, this=this, **extra)
749 with self.open("wb") as f:
750 f.write(rv.encode("utf-8") + b"\n")
752 def _memorize_dependencies(self, dependencies=None, virtual_dependencies=None, for_failure=False):
753 """This updates the dependencies recorded for the artifact based
754 on the direct sources plus the provided dependencies. This also
755 stores the config hash.
757 This normally defers the operation until commit but the `for_failure`
758 more will immediately commit into a new connection.
759 """
761 def operation(con):
762 primary_sources = {self.build_state.to_source_filename(x) for x in self.sources}
764 seen = set()
765 rows = []
766 for source in chain(self.sources, dependencies or ()):
767 source = self.build_state.to_source_filename(source)
768 if source in seen:
769 continue
770 info = self.build_state.get_file_info(source)
771 rows.append(
772 artifacts_row(
773 artifact=self.artifact_name,
774 source=source,
775 source_mtime=info.mtime,
776 source_size=info.size,
777 source_checksum=info.checksum,
778 is_dir=info.is_dir,
779 is_virtual=False,
780 is_primary_source=source in primary_sources,
781 )
782 )
784 seen.add(source)
786 for v_source in virtual_dependencies or ():
787 checksum = v_source.get_checksum(self.build_state.path_cache)
788 mtime = v_source.get_mtime(self.build_state.path_cache)
789 rows.append(
790 artifacts_row(
791 artifact=self.artifact_name,
792 source=_pack_virtual_source_path(v_source.path, v_source.alt),
793 source_mtime=mtime,
794 source_size=None,
795 source_checksum=checksum,
796 is_dir=False,
797 is_virtual=True,
798 is_primary_source=False,
799 )
800 )
802 reporter.report_dependencies(rows)
804 cur = con.cursor()
805 if not for_failure:
806 cur.execute("delete from artifacts where artifact = ?", [self.artifact_name])
807 if rows:
808 cur.executemany(
809 """
810 insert or replace into artifacts (
811 artifact, source, source_mtime, source_size,
812 source_checksum, is_dir, is_virtual, is_primary_source)
813 values (?, ?, ?, ?, ?, ?, ?, ?)
814 """,
815 rows,
816 )
818 if self.config_hash is None:
819 cur.execute(
820 """
821 delete from artifact_config_hashes
822 where artifact = ?
823 """,
824 [self.artifact_name],
825 )
826 else:
827 cur.execute(
828 """
829 insert or replace into artifact_config_hashes
830 (artifact, config_hash) values (?, ?)
831 """,
832 [self.artifact_name, self.config_hash],
833 )
835 cur.close()
837 if for_failure:
838 con = self.build_state.connect_to_database()
839 try:
840 operation(con)
841 except:
842 con.rollback()
843 con.close()
844 raise
845 con.commit()
846 con.close()
847 else:
848 self._auto_deferred_update_operation(operation)
850 def clear_dirty_flag(self):
851 """Clears the dirty flag for all sources."""
853 def operation(con):
854 sources = [self.build_state.to_source_filename(x) for x in self.sources]
855 cur = con.cursor()
856 cur.execute(
857 f"delete from dirty_sources where source in ({_placeholders(sources)})",
858 sources,
859 )
860 cur.close()
861 reporter.report_dirty_flag(False)
863 self._auto_deferred_update_operation(operation)
865 def set_dirty_flag(self):
866 """Given a list of artifacts this will mark all of their sources
867 as dirty so that they will be rebuilt next time.
868 """
870 def operation(con):
871 sources = set()
872 for source in self.sources:
873 sources.add(self.build_state.to_source_filename(source))
875 if not sources:
876 return
878 cur = con.cursor()
879 cur.executemany(
880 """
881 insert or replace into dirty_sources (source) values (?)
882 """,
883 [(x,) for x in sources],
884 )
885 cur.close()
887 reporter.report_dirty_flag(True)
889 self._auto_deferred_update_operation(operation)
891 def _auto_deferred_update_operation(self, f):
892 """Helper that defers an update operation when inside an update
893 block to a later point. Otherwise it's auto committed.
894 """
895 if self.in_update_block:
896 self._pending_update_ops.append(f)
897 return
898 con = self.build_state.connect_to_database()
899 try:
900 f(con)
901 con.commit()
902 except:
903 con.rollback()
904 raise
905 finally:
906 con.close()
908 @contextmanager
909 def update(self):
910 """Opens the artifact for modifications. At the start the dirty
911 flag is cleared out and if the commit goes through without errors it
912 stays cleared. The setting of the dirty flag has to be done by the
913 caller however based on the `exc_info` on the context.
914 """
915 ctx = self.begin_update()
916 try:
917 yield ctx
918 except: # pylint: disable=bare-except # noqa
919 exc_info = sys.exc_info()
920 self.finish_update(ctx, exc_info)
921 else:
922 self.finish_update(ctx)
924 def begin_update(self):
925 """Begins an update block."""
926 if self.in_update_block:
927 raise RuntimeError("Artifact is already open for updates.")
928 self.updated = False
929 ctx = Context(self)
930 ctx.push()
931 self.in_update_block = True
932 self.clear_dirty_flag()
933 return ctx
935 def _commit(self):
936 con = None
937 try:
938 for op in self._pending_update_ops:
939 if con is None:
940 con = self.build_state.connect_to_database()
941 op(con)
943 if self._new_artifact_file is not None:
944 os.replace(self._new_artifact_file, self.dst_filename)
945 self._new_artifact_file = None
947 if con is not None:
948 con.commit()
949 con.close()
950 con = None
952 self.build_state.updated_artifacts.append(self)
953 self.build_state.builder.failure_controller.clear_failure(self.artifact_name)
954 finally:
955 if con is not None:
956 con.rollback()
957 con.close()
959 def _rollback(self):
960 if self._new_artifact_file is not None:
961 try:
962 os.remove(self._new_artifact_file)
963 except OSError:
964 pass
965 self._new_artifact_file = None
966 self._pending_update_ops = []
968 def finish_update(self, ctx, exc_info=None):
969 """Finalizes an update block."""
970 if not self.in_update_block:
971 raise RuntimeError("Artifact is not open for updates.")
972 ctx.pop()
973 self.in_update_block = False
974 self.updated = True
976 # If there was no error, we memoize the dependencies like normal
977 # and then commit our transaction.
978 if exc_info is None:
979 self._memorize_dependencies(
980 ctx.referenced_dependencies,
981 ctx.referenced_virtual_dependencies,
982 )
983 self._commit()
984 return
986 # If an error happened we roll back all changes and record the
987 # stacktrace in two locations: we record it on the context so
988 # that a called can respond to our failure, and we also persist
989 # it so that the dev server can render it out later.
990 self._rollback()
992 # This is a special form of dependency memorization where we do
993 # not prune old dependencies and we just append new ones and we
994 # use a new database connection that immediately commits.
995 self._memorize_dependencies(
996 ctx.referenced_dependencies,
997 ctx.referenced_virtual_dependencies,
998 for_failure=True,
999 )
1001 ctx.exc_info = exc_info
1002 self.build_state.notify_failure(self, exc_info)
1005class PathCache:
1006 def __init__(self, env):
1007 self.file_info_cache = {}
1008 self.source_filename_cache = {}
1009 self.env = env
1011 def to_source_filename(self, filename):
1012 """Given a path somewhere below the environment this will return the
1013 short source filename that is used internally. Unlike the given
1014 path, this identifier is also platform independent.
1015 """
1016 key = filename
1017 rv = self.source_filename_cache.get(key)
1018 if rv is not None:
1019 return rv
1020 folder = os.path.abspath(self.env.root_path)
1021 if isinstance(folder, str) and not isinstance(filename, str):
1022 filename = filename.decode(fs_enc)
1023 filename = os.path.normpath(os.path.join(folder, filename))
1024 if filename.startswith(folder):
1025 filename = filename[len(folder) :].lstrip(os.path.sep)
1026 if os.path.altsep:
1027 filename = filename.lstrip(os.path.altsep)
1028 else:
1029 raise ValueError(f"The given value ({filename!r}) is not below the source folder ({self.env.root_path!r})")
1030 rv = filename.replace(os.path.sep, "/")
1031 self.source_filename_cache[key] = rv
1032 return rv
1034 def get_file_info(self, filename):
1035 """Returns the file info for a given file. This will be cached
1036 on the generator for the lifetime of it. This means that further
1037 accesses to this file info will not cause more IO but it might not
1038 be safe to use the generator after modifications to the original
1039 files have been performed on the outside.
1041 Generally this function can be used to acquire the file info for
1042 any file on the file system but it should onl be used for source
1043 files or carefully for other things.
1045 The filename given can be a source filename.
1046 """
1047 fn = os.path.join(self.env.root_path, filename)
1048 rv = self.file_info_cache.get(fn)
1049 if rv is None:
1050 self.file_info_cache[fn] = rv = FileInfo(self.env, fn)
1051 return rv
1054class Builder:
1055 def __init__(self, pad, destination_path, buildstate_path=None, extra_flags=None):
1056 self.extra_flags = process_extra_flags(extra_flags)
1057 self.pad = pad
1058 self.destination_path = os.path.abspath(os.path.join(pad.db.env.root_path, destination_path))
1059 if buildstate_path:
1060 self.meta_path = buildstate_path
1061 else:
1062 self.meta_path = os.path.join(self.destination_path, ".lektor")
1063 self.failure_controller = FailureController(pad, self.destination_path)
1065 try:
1066 os.makedirs(self.meta_path)
1067 if os.listdir(self.destination_path) != [".lektor"]:
1068 msg = (
1069 f"The build dir {self.destination_path} hasn't been used before, "
1070 "and other files or folders already exist there. "
1071 "If you prune (which normally follows the build step), "
1072 "they will be deleted. Proceed with building?"
1073 )
1074 if not click.confirm(click.style(msg, fg="yellow")):
1075 os.rmdir(self.meta_path)
1076 raise click.Abort()
1077 except OSError:
1078 pass
1080 con = self.connect_to_database()
1081 try:
1082 create_tables(con)
1083 finally:
1084 con.close()
1086 @property
1087 def env(self):
1088 """The environment backing this generator."""
1089 return self.pad.db.env
1091 @property
1092 def buildstate_database_filename(self):
1093 """The filename for the build state database."""
1094 return os.path.join(self.meta_path, "buildstate")
1096 def connect_to_database(self):
1097 con = sqlite3.connect(
1098 self.buildstate_database_filename,
1099 isolation_level=None,
1100 timeout=10,
1101 check_same_thread=False,
1102 )
1103 cur = con.cursor()
1104 cur.execute("pragma journal_mode=WAL")
1105 cur.execute("pragma synchronous=NORMAL")
1106 con.commit()
1107 cur.close()
1108 return con
1110 def touch_site_config(self):
1111 """Touches the site config which typically will trigger a rebuild."""
1112 project_file = self.env.project.project_file
1113 try:
1114 os.utime(project_file)
1115 except OSError:
1116 pass
1118 def find_files(self, query, alt=PRIMARY_ALT, lang=None, limit=50, types=None):
1119 """Returns a list of files that match the query. This requires that
1120 the source info is up to date and is primarily used by the admin to
1121 show files that exist.
1122 """
1123 return find_files(self, query, alt, lang, limit, types)
1125 def new_build_state(self, path_cache=None):
1126 """Creates a new build state."""
1127 if path_cache is None:
1128 path_cache = PathCache(self.env)
1129 return BuildState(self, path_cache)
1131 def get_build_program(self, source, build_state):
1132 """Finds the right build function for the given source file."""
1133 for cls, builder in chain(reversed(self.env.build_programs), reversed(builtin_build_programs)):
1134 if isinstance(source, cls):
1135 return builder(source, build_state)
1136 raise RuntimeError(f"I do not know how to build {source!r}")
1138 def build_artifact(self, artifact, build_func):
1139 """Various parts of the system once they have an artifact and a
1140 function to build it, will invoke this function. This ultimately
1141 is what builds.
1143 The return value is the ctx that was used to build this thing
1144 if it was built, or `None` otherwise.
1145 """
1146 is_current = artifact.is_current
1147 with reporter.build_artifact(artifact, build_func, is_current):
1148 if not is_current:
1149 with artifact.update() as ctx:
1150 # Upon builing anything we record a dependency to the
1151 # project file. This is not ideal but for the moment
1152 # it will ensure that if the file changes we will
1153 # rebuild.
1154 project_file = self.env.project.project_file
1155 if project_file:
1156 ctx.record_dependency(project_file)
1157 build_func(artifact)
1158 return ctx
1159 return None
1161 @staticmethod
1162 def update_source_info(prog, build_state):
1163 """Updates a single source info based on a program. This is done
1164 automatically as part of a build.
1165 """
1166 info = prog.describe_source_record()
1167 if info is not None:
1168 build_state.write_source_info(info)
1170 def prune(self, all=False):
1171 """This cleans up data left in the build folder that does not
1172 correspond to known artifacts.
1173 """
1174 path_cache = PathCache(self.env)
1175 build_state = self.new_build_state(path_cache=path_cache)
1176 with reporter.build(all and "clean" or "prune", self):
1177 self.env.plugin_controller.emit("before-prune", builder=self, all=all)
1179 for aft in build_state.iter_unreferenced_artifacts(all=all):
1180 reporter.report_pruned_artifact(aft)
1181 filename = build_state.get_destination_filename(aft)
1182 prune_file_and_folder(filename, self.destination_path)
1183 build_state.remove_artifact(aft)
1185 build_state.prune_source_infos()
1186 if all:
1187 build_state.vacuum()
1188 self.env.plugin_controller.emit("after-prune", builder=self, all=all)
1190 def build(self, source, path_cache=None):
1191 """Given a source object, builds it."""
1192 build_state = self.new_build_state(path_cache=path_cache)
1193 with reporter.process_source(source):
1194 prog = self.get_build_program(source, build_state)
1195 self.env.plugin_controller.emit(
1196 "before-build",
1197 builder=self,
1198 build_state=build_state,
1199 source=source,
1200 prog=prog,
1201 )
1202 prog.build()
1203 if build_state.updated_artifacts:
1204 self.update_source_info(prog, build_state)
1205 self.env.plugin_controller.emit(
1206 "after-build",
1207 builder=self,
1208 build_state=build_state,
1209 source=source,
1210 prog=prog,
1211 )
1212 return prog, build_state
1214 def get_initial_build_queue(self):
1215 """Returns the initial build queue as deque."""
1216 return deque(self.pad.get_all_roots())
1218 def extend_build_queue(self, queue, prog):
1219 queue.extend(prog.iter_child_sources())
1220 for func in self.env.custom_generators:
1221 queue.extend(func(prog.source) or ())
1223 def build_all(self):
1224 """Builds the entire tree. Returns the number of failures."""
1225 failures = 0
1226 path_cache = PathCache(self.env)
1227 # We keep a dummy connection here that does not do anything which
1228 # helps us with the WAL handling. See #144
1229 con = self.connect_to_database()
1230 try:
1231 with reporter.build("build", self):
1232 self.env.plugin_controller.emit("before-build-all", builder=self)
1233 to_build = self.get_initial_build_queue()
1234 while to_build:
1235 source = to_build.popleft()
1236 prog, build_state = self.build(source, path_cache=path_cache)
1237 self.extend_build_queue(to_build, prog)
1238 failures += len(build_state.failed_artifacts)
1239 self.env.plugin_controller.emit("after-build-all", builder=self)
1240 if failures:
1241 reporter.report_build_all_failure(failures)
1242 return failures
1243 finally:
1244 con.close()
1246 def update_all_source_infos(self):
1247 """Fast way to update all source infos without having to build
1248 everything.
1249 """
1250 build_state = self.new_build_state()
1251 # We keep a dummy connection here that does not do anything which
1252 # helps us with the WAL handling. See #144
1253 con = self.connect_to_database()
1254 try:
1255 with reporter.build("source info update", self):
1256 to_build = self.get_initial_build_queue()
1257 while to_build:
1258 source = to_build.popleft()
1259 with reporter.process_source(source):
1260 prog = self.get_build_program(source, build_state)
1261 self.update_source_info(prog, build_state)
1262 self.extend_build_queue(to_build, prog)
1263 build_state.prune_source_infos()
1264 finally:
1265 con.close()