1 import atexit

2 import contextlib

3 import fnmatch

4 import importlib.util

5 import itertools

6 import os

7 import shutil

8 import sys

9 import uuid

10 import warnings

11 from enum import Enum

12 from errno import EBADF

13 from errno import ELOOP

14 from errno import ENOENT

15 from errno import ENOTDIR

16 from functools import partial

17 from os.path import expanduser

18 from os.path import expandvars

19 from os.path import isabs

20 from os.path import sep

21 from pathlib import Path

22 from pathlib import PurePath

23 from posixpath import sep as posix_sep

24 from types import ModuleType

25 from typing import Callable

26 from typing import Dict

27 from typing import Iterable

28 from typing import Iterator

29 from typing import Optional

30 from typing import Set

31 from typing import TypeVar

32 from typing import Union

33

34 from _pytest.compat import assert_never

35 from _pytest.outcomes import skip

36 from _pytest.warning_types import PytestWarning

37

38 LOCK_TIMEOUT = 60 * 60 * 24 * 3

39

40

41_AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath)

42

43# The following function, variables and comments were

44# copied from cpython 3.9 Lib/pathlib.py file.

45

46# EBADF - guard against macOS `stat` throwing EBADF

47_IGNORED_ERRORS = (ENOENT, ENOTDIR, EBADF, ELOOP)

48

49_IGNORED_WINERRORS = (

50 21, # ERROR_NOT_READY - drive exists but is not accessible

51 1921, # ERROR_CANT_RESOLVE_FILENAME - fix for broken symlink pointing to itself

52)

53

54

55 def _ignore_error(exception):

56 return (

57 getattr(exception, "errno", None) in _IGNORED_ERRORS

58 or getattr(exception, "winerror", None) in _IGNORED_WINERRORS

59 )

60

61

62 def get_lock_path(path: _AnyPurePath) -> _AnyPurePath:

63 return path.joinpath(".lock")

64

65

66 def on_rm_rf_error(func, path: str, exc, *, start_path: Path) -> bool:

67 """Handle known read-only errors during rmtree.

68

69 The returned value is used only by our own tests.

70 """

71 exctype, excvalue = exc[:2]

72

73 # Another process removed the file in the middle of the "rm_rf" (xdist for example).

74 # More context: https://github.com/pytest-dev/pytest/issues/5974#issuecomment-543799018

75 if isinstance(excvalue, FileNotFoundError):

76 return False

77

78 if not isinstance(excvalue, PermissionError):

79 warnings.warn(

80 PytestWarning(f"(rm_rf) error removing {path}\n{exctype}: {excvalue}")

81 )

82 return False

83

84 if func not in (os.rmdir, os.remove, os.unlink):

85 if func not in (os.open,):

86 warnings.warn(

87 PytestWarning(

88 "(rm_rf) unknown function {} when removing {}:\n{}: {}".format(

89 func, path, exctype, excvalue

90 )

91 )

92 )

93 return False

94

95 # Chmod + retry.

96 import stat

97

98 def chmod_rw(p: str) -> None:

99 mode = os.stat(p).st_mode

100 os.chmod(p, mode | stat.S_IRUSR | stat.S_IWUSR)

101

102 # For files, we need to recursively go upwards in the directories to

103 # ensure they all are also writable.

104 p = Path(path)

105 if p.is_file():

106 for parent in p.parents:

107 chmod_rw(str(parent))

108 # Stop when we reach the original path passed to rm_rf.

109 if parent == start_path:

110 break

111 chmod_rw(str(path))

112

113 func(path)

114 return True

115

116

117 def ensure_extended_length_path(path: Path) -> Path:

118 """Get the extended-length version of a path (Windows).

119

120 On Windows, by default, the maximum length of a path (MAX_PATH) is 260

121 characters, and operations on paths longer than that fail. But it is possible

122 to overcome this by converting the path to "extended-length" form before

123 performing the operation:

124 https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation

125

126 On Windows, this function returns the extended-length absolute version of path.

127 On other platforms it returns path unchanged.

128 """

129 if sys.platform.startswith("win32"):

130 path = path.resolve()

131 path = Path(get_extended_length_path_str(str(path)))

132 return path

133

134

135 def get_extended_length_path_str(path: str) -> str:

136 """Convert a path to a Windows extended length path."""

137 long_path_prefix = "\\\\?\\"

138 unc_long_path_prefix = "\\\\?\\UNC\\"

139 if path.startswith((long_path_prefix, unc_long_path_prefix)):

140 return path

141 # UNC

142 if path.startswith("\\\\"):

143 return unc_long_path_prefix + path[2:]

144 return long_path_prefix + path

145

146

147 def rm_rf(path: Path) -> None:

148 """Remove the path contents recursively, even if some elements

149 are read-only."""

150 path = ensure_extended_length_path(path)

151 onerror = partial(on_rm_rf_error, start_path=path)

152 shutil.rmtree(str(path), onerror=onerror)

153

154

155 def find_prefixed(root: Path, prefix: str) -> Iterator[Path]:

156 """Find all elements in root that begin with the prefix, case insensitive."""

157 l_prefix = prefix.lower()

158 for x in root.iterdir():

159 if x.name.lower().startswith(l_prefix):

160 yield x

161

162

163 def extract_suffixes(iter: Iterable[PurePath], prefix: str) -> Iterator[str]:

164 """Return the parts of the paths following the prefix.

165

166 :param iter: Iterator over path names.

167 :param prefix: Expected prefix of the path names.

168 """

169 p_len = len(prefix)

170 for p in iter:

171 yield p.name[p_len:]

172

173

174 def find_suffixes(root: Path, prefix: str) -> Iterator[str]:

175 """Combine find_prefixes and extract_suffixes."""

176 return extract_suffixes(find_prefixed(root, prefix), prefix)

177

178

179 def parse_num(maybe_num) -> int:

180 """Parse number path suffixes, returns -1 on error."""

181 try:

182 return int(maybe_num)

183 except ValueError:

184 return -1

185

186

187 def _force_symlink(

188 root: Path, target: Union[str, PurePath], link_to: Union[str, Path]

189) -> None:

190 """Helper to create the current symlink.

191

192 It's full of race conditions that are reasonably OK to ignore

193 for the context of best effort linking to the latest test run.

194

195 The presumption being that in case of much parallelism

196 the inaccuracy is going to be acceptable.

197 """

198 current_symlink = root.joinpath(target)

199 try:

200 current_symlink.unlink()

201 except OSError:

202 pass

203 try:

204 current_symlink.symlink_to(link_to)

205 except Exception:

206 pass

207

208

209 def make_numbered_dir(root: Path, prefix: str, mode: int = 0o700) -> Path:

210 """Create a directory with an increased number as suffix for the given prefix."""

211 for i in range(10):

212 # try up to 10 times to create the folder

213 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)

214 new_number = max_existing + 1

215 new_path = root.joinpath(f"{prefix}{new_number}")

216 try:

217 new_path.mkdir(mode=mode)

218 except Exception:

219 pass

220 else:

221 _force_symlink(root, prefix + "current", new_path)

222 return new_path

223 else:

224 raise OSError(

225 "could not create numbered dir with prefix "

226 "{prefix} in {root} after 10 tries".format(prefix=prefix, root=root)

227 )

228

229

230 def create_cleanup_lock(p: Path) -> Path:

231 """Create a lock to prevent premature folder cleanup."""

232 lock_path = get_lock_path(p)

233 try:

234 fd = os.open(str(lock_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)

235 except FileExistsError as e:

236 raise OSError(f"cannot create lockfile in {p}") from e

237 else:

238 pid = os.getpid()

239 spid = str(pid).encode()

240 os.write(fd, spid)

241 os.close(fd)

242 if not lock_path.is_file():

243 raise OSError("lock path got renamed after successful creation")

244 return lock_path

245

246

247 def register_cleanup_lock_removal(lock_path: Path, register=atexit.register):

248 """Register a cleanup function for removing a lock, by default on atexit."""

249 pid = os.getpid()

250

251 def cleanup_on_exit(lock_path: Path = lock_path, original_pid: int = pid) -> None:

252 current_pid = os.getpid()

253 if current_pid != original_pid:

254 # fork

255 return

256 try:

257 lock_path.unlink()

258 except OSError:

259 pass

260

261 return register(cleanup_on_exit)

262

263

264 def maybe_delete_a_numbered_dir(path: Path) -> None:

265 """Remove a numbered directory if its lock can be obtained and it does

266 not seem to be in use."""

267 path = ensure_extended_length_path(path)

268 lock_path = None

269 try:

270 lock_path = create_cleanup_lock(path)

271 parent = path.parent

272

273 garbage = parent.joinpath(f"garbage-{uuid.uuid4()}")

274 path.rename(garbage)

275 rm_rf(garbage)

276 except OSError:

277 # known races:

278 # * other process did a cleanup at the same time

279 # * deletable folder was found

280 # * process cwd (Windows)

281 return

282 finally:

283 # If we created the lock, ensure we remove it even if we failed

284 # to properly remove the numbered dir.

285 if lock_path is not None:

286 try:

287 lock_path.unlink()

288 except OSError:

289 pass

290

291

292 def ensure_deletable(path: Path, consider_lock_dead_if_created_before: float) -> bool:

293 """Check if `path` is deletable based on whether the lock file is expired."""

294 if path.is_symlink():

295 return False

296 lock = get_lock_path(path)

297 try:

298 if not lock.is_file():

299 return True

300 except OSError:

301 # we might not have access to the lock file at all, in this case assume

302 # we don't have access to the entire directory (#7491).

303 return False

304 try:

305 lock_time = lock.stat().st_mtime

306 except Exception:

307 return False

308 else:

309 if lock_time < consider_lock_dead_if_created_before:

310 # We want to ignore any errors while trying to remove the lock such as:

311 # - PermissionDenied, like the file permissions have changed since the lock creation;

312 # - FileNotFoundError, in case another pytest process got here first;

313 # and any other cause of failure.

314 with contextlib.suppress(OSError):

315 lock.unlink()

316 return True

317 return False

318

319

320 def try_cleanup(path: Path, consider_lock_dead_if_created_before: float) -> None:

321 """Try to cleanup a folder if we can ensure it's deletable."""

322 if ensure_deletable(path, consider_lock_dead_if_created_before):

323 maybe_delete_a_numbered_dir(path)

324

325

326 def cleanup_candidates(root: Path, prefix: str, keep: int) -> Iterator[Path]:

327 """List candidates for numbered directories to be removed - follows py.path."""

328 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)

329 max_delete = max_existing - keep

330 paths = find_prefixed(root, prefix)

331 paths, paths2 = itertools.tee(paths)

332 numbers = map(parse_num, extract_suffixes(paths2, prefix))

333 for path, number in zip(paths, numbers):

334 if number <= max_delete:

335 yield path

336

337

338 def cleanup_numbered_dir(

339 root: Path, prefix: str, keep: int, consider_lock_dead_if_created_before: float

340) -> None:

341 """Cleanup for lock driven numbered directories."""

342 for path in cleanup_candidates(root, prefix, keep):

343 try_cleanup(path, consider_lock_dead_if_created_before)

344 for path in root.glob("garbage-*"):

345 try_cleanup(path, consider_lock_dead_if_created_before)

346

347

348 def make_numbered_dir_with_cleanup(

349 root: Path,

350 prefix: str,

351 keep: int,

352 lock_timeout: float,

353 mode: int,

354) -> Path:

355 """Create a numbered dir with a cleanup lock and remove old ones."""

356 e = None

357 for i in range(10):

358 try:

359 p = make_numbered_dir(root, prefix, mode)

360 lock_path = create_cleanup_lock(p)

361 register_cleanup_lock_removal(lock_path)

362 except Exception as exc:

363 e = exc

364 else:

365 consider_lock_dead_if_created_before = p.stat().st_mtime - lock_timeout

366 # Register a cleanup for program exit

367 atexit.register(

368 cleanup_numbered_dir,

369 root,

370 prefix,

371 keep,

372 consider_lock_dead_if_created_before,

373 )

374 return p

375 assert e is not None

376 raise e

377

378

379 def resolve_from_str(input: str, rootpath: Path) -> Path:

380 input = expanduser(input)

381 input = expandvars(input)

382 if isabs(input):

383 return Path(input)

384 else:

385 return rootpath.joinpath(input)

386

387

388 def fnmatch_ex(pattern: str, path: Union[str, "os.PathLike[str]"]) -> bool:

389 """A port of FNMatcher from py.path.common which works with PurePath() instances.

390

391 The difference between this algorithm and PurePath.match() is that the

392 latter matches "**" glob expressions for each part of the path, while

393 this algorithm uses the whole path instead.

394

395 For example:

396 "tests/foo/bar/doc/test_foo.py" matches pattern "tests/**/doc/test*.py"

397 with this algorithm, but not with PurePath.match().

398

399 This algorithm was ported to keep backward-compatibility with existing

400 settings which assume paths match according this logic.

401

402 References:

403 * https://bugs.python.org/issue29249

404 * https://bugs.python.org/issue34731

405 """

406 path = PurePath(path)

407 iswin32 = sys.platform.startswith("win")

408

409 if iswin32 and sep not in pattern and posix_sep in pattern:

410 # Running on Windows, the pattern has no Windows path separators,

411 # and the pattern has one or more Posix path separators. Replace

412 # the Posix path separators with the Windows path separator.

413 pattern = pattern.replace(posix_sep, sep)

414

415 if sep not in pattern:

416 name = path.name

417 else:

418 name = str(path)

419 if path.is_absolute() and not os.path.isabs(pattern):

420 pattern = f"*{os.sep}{pattern}"

421 return fnmatch.fnmatch(name, pattern)

422

423

424 def parts(s: str) -> Set[str]:

425 parts = s.split(sep)

426 return {sep.join(parts[: i + 1]) or sep for i in range(len(parts))}

427

428

429 def symlink_or_skip(src, dst, **kwargs):

430 """Make a symlink, or skip the test in case symlinks are not supported."""

431 try:

432 os.symlink(str(src), str(dst), **kwargs)

433 except OSError as e:

434 skip(f"symlinks not supported: {e}")

435

436

437 class ImportMode(Enum):

438 """Possible values for `mode` parameter of `import_path`."""

439

440 prepend = "prepend"

441 append = "append"

442 importlib = "importlib"

443

444

445 class ImportPathMismatchError(ImportError):

446 """Raised on import_path() if there is a mismatch of __file__'s.

447

448 This can happen when `import_path` is called multiple times with different filenames that has

449 the same basename but reside in packages

450 (for example "/tests1/test_foo.py" and "/tests2/test_foo.py").

451 """

452

453

454 def import_path(

455 p: Union[str, "os.PathLike[str]"],

456 *,

457 mode: Union[str, ImportMode] = ImportMode.prepend,

458 root: Path,

459) -> ModuleType:

460 """Import and return a module from the given path, which can be a file (a module) or

461 a directory (a package).

462

463 The import mechanism used is controlled by the `mode` parameter:

464

465 * `mode == ImportMode.prepend`: the directory containing the module (or package, taking

466 `__init__.py` files into account) will be put at the *start* of `sys.path` before

467 being imported with `importlib.import_module`.

468

469 * `mode == ImportMode.append`: same as `prepend`, but the directory will be appended

470 to the end of `sys.path`, if not already in `sys.path`.

471

472 * `mode == ImportMode.importlib`: uses more fine control mechanisms provided by `importlib`

473 to import the module, which avoids having to muck with `sys.path` at all. It effectively

474 allows having same-named test modules in different places.

475

476 :param root:

477 Used as an anchor when mode == ImportMode.importlib to obtain

478 a unique name for the module being imported so it can safely be stored

479 into ``sys.modules``.

480

481 :raises ImportPathMismatchError:

482 If after importing the given `path` and the module `__file__`

483 are different. Only raised in `prepend` and `append` modes.

484 """

485 mode = ImportMode(mode)

486

487 path = Path(p)

488

489 if not path.exists():

490 raise ImportError(path)

491

492 if mode is ImportMode.importlib:

493 module_name = module_name_from_path(path, root)

494

495 for meta_importer in sys.meta_path:

496 spec = meta_importer.find_spec(module_name, [str(path.parent)])

497 if spec is not None:

498 break

499 else:

500 spec = importlib.util.spec_from_file_location(module_name, str(path))

501

502 if spec is None:

503 raise ImportError(f"Can't find module {module_name} at location {path}")

504 mod = importlib.util.module_from_spec(spec)

505 sys.modules[module_name] = mod

506 spec.loader.exec_module(mod) # type: ignore[union-attr]

507 insert_missing_modules(sys.modules, module_name)

508 return mod

509

510 pkg_path = resolve_package_path(path)

511 if pkg_path is not None:

512 pkg_root = pkg_path.parent

513 names = list(path.with_suffix("").relative_to(pkg_root).parts)

514 if names[-1] == "__init__":

515 names.pop()

516 module_name = ".".join(names)

517 else:

518 pkg_root = path.parent

519 module_name = path.stem

520

521 # Change sys.path permanently: restoring it at the end of this function would cause surprising

522 # problems because of delayed imports: for example, a conftest.py file imported by this function

523 # might have local imports, which would fail at runtime if we restored sys.path.

524 if mode is ImportMode.append:

525 if str(pkg_root) not in sys.path:

526 sys.path.append(str(pkg_root))

527 elif mode is ImportMode.prepend:

528 if str(pkg_root) != sys.path[0]:

529 sys.path.insert(0, str(pkg_root))

530 else:

531 assert_never(mode)

532

533 importlib.import_module(module_name)

534

535 mod = sys.modules[module_name]

536 if path.name == "__init__.py":

537 return mod

538

539 ignore = os.environ.get("PY_IGNORE_IMPORTMISMATCH", "")

540 if ignore != "1":

541 module_file = mod.__file__

542 if module_file is None:

543 raise ImportPathMismatchError(module_name, module_file, path)

544

545 if module_file.endswith((".pyc", ".pyo")):

546 module_file = module_file[:-1]

547 if module_file.endswith(os.path.sep + "__init__.py"):

548 module_file = module_file[: -(len(os.path.sep + "__init__.py"))]

549

550 try:

551 is_same = _is_same(str(path), module_file)

552 except FileNotFoundError:

553 is_same = False

554

555 if not is_same:

556 raise ImportPathMismatchError(module_name, module_file, path)

557

558 return mod

559

560

561# Implement a special _is_same function on Windows which returns True if the two filenames

562# compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678).

563 if sys.platform.startswith("win"):

564

565 def _is_same(f1: str, f2: str) -> bool:

566 return Path(f1) == Path(f2) or os.path.samefile(f1, f2)

567

568 else:

569

570 def _is_same(f1: str, f2: str) -> bool:

571 return os.path.samefile(f1, f2)

572

573

574 def module_name_from_path(path: Path, root: Path) -> str:

575 """

576 Return a dotted module name based on the given path, anchored on root.

577

578 For example: path="projects/src/tests/test_foo.py" and root="/projects", the

579 resulting module name will be "src.tests.test_foo".

580 """

581 path = path.with_suffix("")

582 try:

583 relative_path = path.relative_to(root)

584 except ValueError:

585 # If we can't get a relative path to root, use the full path, except

586 # for the first part ("d:\\" or "/" depending on the platform, for example).

587 path_parts = path.parts[1:]

588 else:

589 # Use the parts for the relative path to the root path.

590 path_parts = relative_path.parts

591

592 return ".".join(path_parts)

593

594

595 def insert_missing_modules(modules: Dict[str, ModuleType], module_name: str) -> None:

596 """

597 Used by ``import_path`` to create intermediate modules when using mode=importlib.

598

599 When we want to import a module as "src.tests.test_foo" for example, we need

600 to create empty modules "src" and "src.tests" after inserting "src.tests.test_foo",

601 otherwise "src.tests.test_foo" is not importable by ``__import__``.

602 """

603 module_parts = module_name.split(".")

604 while module_name:

605 if module_name not in modules:

606 try:

607 # If sys.meta_path is empty, calling import_module will issue

608 # a warning and raise ModuleNotFoundError. To avoid the

609 # warning, we check sys.meta_path explicitly and raise the error

610 # ourselves to fall back to creating a dummy module.

611 if not sys.meta_path:

612 raise ModuleNotFoundError

613 importlib.import_module(module_name)

614 except ModuleNotFoundError:

615 module = ModuleType(

616 module_name,

617 doc="Empty module created by pytest's importmode=importlib.",

618 )

619 modules[module_name] = module

620 module_parts.pop(-1)

621 module_name = ".".join(module_parts)

622

623

624 def resolve_package_path(path: Path) -> Optional[Path]:

625 """Return the Python package path by looking for the last

626 directory upwards which still contains an __init__.py.

627

628 Returns None if it can not be determined.

629 """

630 result = None

631 for parent in itertools.chain((path,), path.parents):

632 if parent.is_dir():

633 if not parent.joinpath("__init__.py").is_file():

634 break

635 if not parent.name.isidentifier():

636 break

637 result = parent

638 return result

639

640

641 def visit(

642 path: Union[str, "os.PathLike[str]"], recurse: Callable[["os.DirEntry[str]"], bool]

643) -> Iterator["os.DirEntry[str]"]:

644 """Walk a directory recursively, in breadth-first order.

645

646 Entries at each directory level are sorted.

647 """

648

649 # Skip entries with symlink loops and other brokenness, so the caller doesn't

650 # have to deal with it.

651 entries = []

652 for entry in os.scandir(path):

653 try:

654 entry.is_file()

655 except OSError as err:

656 if _ignore_error(err):

657 continue

658 raise

659 entries.append(entry)

660

661 entries.sort(key=lambda entry: entry.name)

662

663 yield from entries

664

665 for entry in entries:

666 if entry.is_dir() and recurse(entry):

667 yield from visit(entry.path, recurse)

668

669

670 def absolutepath(path: Union[Path, str]) -> Path:

671 """Convert a path to an absolute path using os.path.abspath.

672

673 Prefer this over Path.resolve() (see #6523).

674 Prefer this over Path.absolute() (not public, doesn't normalize).

675 """

676 return Path(os.path.abspath(str(path)))

677

678

679 def commonpath(path1: Path, path2: Path) -> Optional[Path]:

680 """Return the common part shared with the other path, or None if there is

681 no common part.

682

683 If one path is relative and one is absolute, returns None.

684 """

685 try:

686 return Path(os.path.commonpath((str(path1), str(path2))))

687 except ValueError:

688 return None

689

690

691 def bestrelpath(directory: Path, dest: Path) -> str:

692 """Return a string which is a relative path from directory to dest such

693 that directory/bestrelpath == dest.

694

695 The paths must be either both absolute or both relative.

696

697 If no such path can be determined, returns dest.

698 """

699 assert isinstance(directory, Path)

700 assert isinstance(dest, Path)

701 if dest == directory:

702 return os.curdir

703 # Find the longest common directory.

704 base = commonpath(directory, dest)

705 # Can be the case on Windows for two absolute paths on different drives.

706 # Can be the case for two relative paths without common prefix.

707 # Can be the case for a relative path and an absolute path.

708 if not base:

709 return str(dest)

710 reldirectory = directory.relative_to(base)

711 reldest = dest.relative_to(base)

712 return os.path.join(

713 # Back from directory to base.

714 *([os.pardir] * len(reldirectory.parts)),

715 # Forward from base to dest.

716 *reldest.parts,

717 )

718

719

720# Originates from py. path.local.copy(), with siginficant trims and adjustments.

721# TODO(py38): Replace with shutil.copytree(..., symlinks=True, dirs_exist_ok=True)

722 def copytree(source: Path, target: Path) -> None:

723 """Recursively copy a source directory to target."""

724 assert source.is_dir()

725 for entry in visit(source, recurse=lambda entry: not entry.is_symlink()):

726 x = Path(entry)

727 relpath = x.relative_to(source)

728 newx = target / relpath

729 newx.parent.mkdir(exist_ok=True)

730 if x.is_symlink():

731 newx.symlink_to(os.readlink(x))

732 elif x.is_file():

733 shutil.copyfile(x, newx)

734 elif x.is_dir():

735 newx.mkdir(exist_ok=True)