init

.venv/lib/python3.8/site-packages/virtualenv/seed/embed/base_embed.py

@@ -0,0 +1,118 @@
+from __future__ import absolute_import, unicode_literals
+
+from abc import ABCMeta
+
+from six import add_metaclass
+
+from virtualenv.util.path import Path
+from virtualenv.util.six import ensure_str, ensure_text
+
+from ..seeder import Seeder
+from ..wheels import Version
+
+PERIODIC_UPDATE_ON_BY_DEFAULT = True
+
+
+@add_metaclass(ABCMeta)
+class BaseEmbed(Seeder):
+    def __init__(self, options):
+        super(BaseEmbed, self).__init__(options, enabled=options.no_seed is False)
+
+        self.download = options.download
+        self.extra_search_dir = [i.resolve() for i in options.extra_search_dir if i.exists()]
+
+        self.pip_version = options.pip
+        self.setuptools_version = options.setuptools
+        self.wheel_version = options.wheel
+
+        self.no_pip = options.no_pip
+        self.no_setuptools = options.no_setuptools
+        self.no_wheel = options.no_wheel
+        self.app_data = options.app_data
+        self.periodic_update = not options.no_periodic_update
+
+        if not self.distribution_to_versions():
+            self.enabled = False
+
+    @classmethod
+    def distributions(cls):
+        return {
+            "pip": Version.bundle,
+            "setuptools": Version.bundle,
+            "wheel": Version.bundle,
+        }
+
+    def distribution_to_versions(self):
+        return {
+            distribution: getattr(self, "{}_version".format(distribution))
+            for distribution in self.distributions()
+            if getattr(self, "no_{}".format(distribution)) is False
+        }
+
+    @classmethod
+    def add_parser_arguments(cls, parser, interpreter, app_data):
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument(
+            "--no-download",
+            "--never-download",
+            dest="download",
+            action="store_false",
+            help="pass to disable download of the latest {} from PyPI".format("/".join(cls.distributions())),
+            default=True,
+        )
+        group.add_argument(
+            "--download",
+            dest="download",
+            action="store_true",
+            help="pass to enable download of the latest {} from PyPI".format("/".join(cls.distributions())),
+            default=False,
+        )
+        parser.add_argument(
+            "--extra-search-dir",
+            metavar="d",
+            type=Path,
+            nargs="+",
+            help="a path containing wheels to extend the internal wheel list (can be set 1+ times)",
+            default=[],
+        )
+        for distribution, default in cls.distributions().items():
+            parser.add_argument(
+                "--{}".format(distribution),
+                dest=distribution,
+                metavar="version",
+                help="version of {} to install as seed: embed, bundle or exact version".format(distribution),
+                default=default,
+            )
+        for distribution in cls.distributions():
+            parser.add_argument(
+                "--no-{}".format(distribution),
+                dest="no_{}".format(distribution),
+                action="store_true",
+                help="do not install {}".format(distribution),
+                default=False,
+            )
+        parser.add_argument(
+            "--no-periodic-update",
+            dest="no_periodic_update",
+            action="store_true",
+            help="disable the periodic (once every 14 days) update of the embedded wheels",
+            default=not PERIODIC_UPDATE_ON_BY_DEFAULT,
+        )
+
+    def __unicode__(self):
+        result = self.__class__.__name__
+        result += "("
+        if self.extra_search_dir:
+            result += "extra_search_dir={},".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir))
+        result += "download={},".format(self.download)
+        for distribution in self.distributions():
+            if getattr(self, "no_{}".format(distribution)):
+                continue
+            result += " {}{},".format(
+                distribution,
+                "={}".format(getattr(self, "{}_version".format(distribution), None) or "latest"),
+            )
+        return result[:-1] + ")"
+
+    def __repr__(self):
+        return ensure_str(self.__unicode__())

.venv/lib/python3.8/site-packages/virtualenv/seed/embed/pip_invoke.py

@@ -0,0 +1,57 @@
+from __future__ import absolute_import, unicode_literals
+
+import logging
+from contextlib import contextmanager
+
+from virtualenv.discovery.cached_py_info import LogCmd
+from virtualenv.seed.embed.base_embed import BaseEmbed
+from virtualenv.util.subprocess import Popen
+
+from ..wheels import Version, get_wheel, pip_wheel_env_run
+
+
+class PipInvoke(BaseEmbed):
+    def __init__(self, options):
+        super(PipInvoke, self).__init__(options)
+
+    def run(self, creator):
+        if not self.enabled:
+            return
+        for_py_version = creator.interpreter.version_release_str
+        with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd:
+            env = pip_wheel_env_run(self.extra_search_dir, self.app_data, self.env)
+            self._execute(cmd, env)
+
+    @staticmethod
+    def _execute(cmd, env):
+        logging.debug("pip seed by running: %s", LogCmd(cmd, env))
+        process = Popen(cmd, env=env)
+        process.communicate()
+        if process.returncode != 0:
+            raise RuntimeError("failed seed with code {}".format(process.returncode))
+        return process
+
+    @contextmanager
+    def get_pip_install_cmd(self, exe, for_py_version):
+        cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:", "--disable-pip-version-check"]
+        if not self.download:
+            cmd.append("--no-index")
+        folders = set()
+        for dist, version in self.distribution_to_versions().items():
+            wheel = get_wheel(
+                distribution=dist,
+                version=version,
+                for_py_version=for_py_version,
+                search_dirs=self.extra_search_dir,
+                download=False,
+                app_data=self.app_data,
+                do_periodic_update=self.periodic_update,
+                env=self.env,
+            )
+            if wheel is None:
+                raise RuntimeError("could not get wheel for distribution {}".format(dist))
+            folders.add(str(wheel.path.parent))
+            cmd.append(Version.as_pip_req(dist, wheel.version))
+        for folder in sorted(folders):
+            cmd.extend(["--find-links", str(folder)])
+        yield cmd

.venv/lib/python3.8/site-packages/virtualenv/seed/embed/via_app_data/pip_install/base.py

@@ -0,0 +1,200 @@
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import re
+import zipfile
+from abc import ABCMeta, abstractmethod
+from itertools import chain
+from tempfile import mkdtemp
+
+from distlib.scripts import ScriptMaker, enquote_executable
+from six import PY3, add_metaclass
+
+from virtualenv.util import ConfigParser
+from virtualenv.util.path import Path, safe_delete
+from virtualenv.util.six import ensure_text
+
+
+@add_metaclass(ABCMeta)
+class PipInstall(object):
+    def __init__(self, wheel, creator, image_folder):
+        self._wheel = wheel
+        self._creator = creator
+        self._image_dir = image_folder
+        self._extracted = False
+        self.__dist_info = None
+        self._console_entry_points = None
+
+    @abstractmethod
+    def _sync(self, src, dst):
+        raise NotImplementedError
+
+    def install(self, version_info):
+        self._extracted = True
+        self._uninstall_previous_version()
+        # sync image
+        for filename in self._image_dir.iterdir():
+            into = self._creator.purelib / filename.name
+            self._sync(filename, into)
+        # generate console executables
+        consoles = set()
+        script_dir = self._creator.script_dir
+        for name, module in self._console_scripts.items():
+            consoles.update(self._create_console_entry_point(name, module, script_dir, version_info))
+        logging.debug("generated console scripts %s", " ".join(i.name for i in consoles))
+
+    def build_image(self):
+        # 1. first extract the wheel
+        logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir)
+        with zipfile.ZipFile(str(self._wheel)) as zip_ref:
+            self._shorten_path_if_needed(zip_ref)
+            zip_ref.extractall(str(self._image_dir))
+            self._extracted = True
+        # 2. now add additional files not present in the distribution
+        new_files = self._generate_new_files()
+        # 3. finally fix the records file
+        self._fix_records(new_files)
+
+    def _shorten_path_if_needed(self, zip_ref):
+        if os.name == "nt":
+            to_folder = str(self._image_dir)
+            # https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
+            zip_max_len = max(len(i) for i in zip_ref.namelist())
+            path_len = zip_max_len + len(to_folder)
+            if path_len > 260:
+                self._image_dir.mkdir(exist_ok=True)  # to get a short path must exist
+
+                from virtualenv.util.path import get_short_path_name
+
+                to_folder = get_short_path_name(to_folder)
+                self._image_dir = Path(to_folder)
+
+    def _records_text(self, files):
+        record_data = "\n".join(
+            "{},,".format(os.path.relpath(ensure_text(str(rec)), ensure_text(str(self._image_dir)))) for rec in files
+        )
+        return record_data
+
+    def _generate_new_files(self):
+        new_files = set()
+        installer = self._dist_info / "INSTALLER"
+        installer.write_text("pip\n")
+        new_files.add(installer)
+        # inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226
+        marker = self._image_dir / "{}.virtualenv".format(self._dist_info.stem)
+        marker.write_text("")
+        new_files.add(marker)
+        folder = mkdtemp()
+        try:
+            to_folder = Path(folder)
+            rel = os.path.relpath(ensure_text(str(self._creator.script_dir)), ensure_text(str(self._creator.purelib)))
+            version_info = self._creator.interpreter.version_info
+            for name, module in self._console_scripts.items():
+                new_files.update(
+                    Path(os.path.normpath(ensure_text(str(self._image_dir / rel / i.name))))
+                    for i in self._create_console_entry_point(name, module, to_folder, version_info)
+                )
+        finally:
+            safe_delete(folder)
+        return new_files
+
+    @property
+    def _dist_info(self):
+        if self._extracted is False:
+            return None  # pragma: no cover
+        if self.__dist_info is None:
+            files = []
+            for filename in self._image_dir.iterdir():
+                files.append(filename.name)
+                if filename.suffix == ".dist-info":
+                    self.__dist_info = filename
+                    break
+            else:
+                msg = "no .dist-info at {}, has {}".format(self._image_dir, ", ".join(files))  # pragma: no cover
+                raise RuntimeError(msg)  # pragma: no cover
+        return self.__dist_info
+
+    @abstractmethod
+    def _fix_records(self, extra_record_data):
+        raise NotImplementedError
+
+    @property
+    def _console_scripts(self):
+        if self._extracted is False:
+            return None  # pragma: no cover
+        if self._console_entry_points is None:
+            self._console_entry_points = {}
+            entry_points = self._dist_info / "entry_points.txt"
+            if entry_points.exists():
+                parser = ConfigParser.ConfigParser()
+                with entry_points.open() as file_handler:
+                    reader = getattr(parser, "read_file" if PY3 else "readfp")
+                    reader(file_handler)
+                if "console_scripts" in parser.sections():
+                    for name, value in parser.items("console_scripts"):
+                        match = re.match(r"(.*?)-?\d\.?\d*", name)
+                        if match:
+                            name = match.groups(1)[0]
+                        self._console_entry_points[name] = value
+        return self._console_entry_points
+
+    def _create_console_entry_point(self, name, value, to_folder, version_info):
+        result = []
+        maker = ScriptMakerCustom(to_folder, version_info, self._creator.exe, name)
+        specification = "{} = {}".format(name, value)
+        new_files = maker.make(specification)
+        result.extend(Path(i) for i in new_files)
+        return result
+
+    def _uninstall_previous_version(self):
+        dist_name = self._dist_info.stem.split("-")[0]
+        in_folders = chain.from_iterable([i.iterdir() for i in {self._creator.purelib, self._creator.platlib}])
+        paths = (p for p in in_folders if p.stem.split("-")[0] == dist_name and p.suffix == ".dist-info" and p.is_dir())
+        existing_dist = next(paths, None)
+        if existing_dist is not None:
+            self._uninstall_dist(existing_dist)
+
+    @staticmethod
+    def _uninstall_dist(dist):
+        dist_base = dist.parent
+        logging.debug("uninstall existing distribution %s from %s", dist.stem, dist_base)
+
+        top_txt = dist / "top_level.txt"  # add top level packages at folder level
+        paths = {dist.parent / i.strip() for i in top_txt.read_text().splitlines()} if top_txt.exists() else set()
+        paths.add(dist)  # add the dist-info folder itself
+
+        base_dirs, record = paths.copy(), dist / "RECORD"  # collect entries in record that we did not register yet
+        for name in (i.split(",")[0] for i in record.read_text().splitlines()) if record.exists() else ():
+            path = dist_base / name
+            if not any(p in base_dirs for p in path.parents):  # only add if not already added as a base dir
+                paths.add(path)
+
+        for path in sorted(paths):  # actually remove stuff in a stable order
+            if path.exists():
+                if path.is_dir() and not path.is_symlink():
+                    safe_delete(path)
+                else:
+                    path.unlink()
+
+    def clear(self):
+        if self._image_dir.exists():
+            safe_delete(self._image_dir)
+
+    def has_image(self):
+        return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None
+
+
+class ScriptMakerCustom(ScriptMaker):
+    def __init__(self, target_dir, version_info, executable, name):
+        super(ScriptMakerCustom, self).__init__(None, str(target_dir))
+        self.clobber = True  # overwrite
+        self.set_mode = True  # ensure they are executable
+        self.executable = enquote_executable(str(executable))
+        self.version_info = version_info.major, version_info.minor
+        self.variants = {"", "X", "X.Y"}
+        self._name = name
+
+    def _write_script(self, names, shebang, script_bytes, filenames, ext):
+        names.add("{}{}.{}".format(self._name, *self.version_info))
+        super(ScriptMakerCustom, self)._write_script(names, shebang, script_bytes, filenames, ext)

.venv/lib/python3.8/site-packages/virtualenv/seed/embed/via_app_data/pip_install/copy.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+from virtualenv.util.path import Path, copy
+from virtualenv.util.six import ensure_text
+
+from .base import PipInstall
+
+
+class CopyPipInstall(PipInstall):
+    def _sync(self, src, dst):
+        copy(src, dst)
+
+    def _generate_new_files(self):
+        # create the pyc files
+        new_files = super(CopyPipInstall, self)._generate_new_files()
+        new_files.update(self._cache_files())
+        return new_files
+
+    def _cache_files(self):
+        version = self._creator.interpreter.version_info
+        py_c_ext = ".{}-{}{}.pyc".format(self._creator.interpreter.implementation.lower(), version.major, version.minor)
+        for root, dirs, files in os.walk(ensure_text(str(self._image_dir)), topdown=True):
+            root_path = Path(root)
+            for name in files:
+                if name.endswith(".py"):
+                    yield root_path / "{}{}".format(name[:-3], py_c_ext)
+            for name in dirs:
+                yield root_path / name / "__pycache__"
+
+    def _fix_records(self, new_files):
+        extra_record_data_str = self._records_text(new_files)
+        with open(ensure_text(str(self._dist_info / "RECORD")), "ab") as file_handler:
+            file_handler.write(extra_record_data_str.encode("utf-8"))

.venv/lib/python3.8/site-packages/virtualenv/seed/embed/via_app_data/pip_install/symlink.py

@@ -0,0 +1,61 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+import subprocess
+from stat import S_IREAD, S_IRGRP, S_IROTH
+
+from virtualenv.util.path import safe_delete, set_tree
+from virtualenv.util.six import ensure_text
+from virtualenv.util.subprocess import Popen
+
+from .base import PipInstall
+
+
+class SymlinkPipInstall(PipInstall):
+    def _sync(self, src, dst):
+        src_str = ensure_text(str(src))
+        dest_str = ensure_text(str(dst))
+        os.symlink(src_str, dest_str)
+
+    def _generate_new_files(self):
+        # create the pyc files, as the build image will be R/O
+        process = Popen(
+            [ensure_text(str(self._creator.exe)), "-m", "compileall", ensure_text(str(self._image_dir))],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        process.communicate()
+        # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for close
+        root_py_cache = self._image_dir / "__pycache__"
+        new_files = set()
+        if root_py_cache.exists():
+            new_files.update(root_py_cache.iterdir())
+            new_files.add(root_py_cache)
+            safe_delete(root_py_cache)
+        core_new_files = super(SymlinkPipInstall, self)._generate_new_files()
+        # remove files that are within the image folder deeper than one level (as these will be not linked directly)
+        for file in core_new_files:
+            try:
+                rel = file.relative_to(self._image_dir)
+                if len(rel.parts) > 1:
+                    continue
+            except ValueError:
+                pass
+            new_files.add(file)
+        return new_files
+
+    def _fix_records(self, new_files):
+        new_files.update(i for i in self._image_dir.iterdir())
+        extra_record_data_str = self._records_text(sorted(new_files, key=str))
+        with open(ensure_text(str(self._dist_info / "RECORD")), "wb") as file_handler:
+            file_handler.write(extra_record_data_str.encode("utf-8"))
+
+    def build_image(self):
+        super(SymlinkPipInstall, self).build_image()
+        # protect the image by making it read only
+        set_tree(self._image_dir, S_IREAD | S_IRGRP | S_IROTH)
+
+    def clear(self):
+        if self._image_dir.exists():
+            safe_delete(self._image_dir)
+        super(SymlinkPipInstall, self).clear()

.venv/lib/python3.8/site-packages/virtualenv/seed/embed/via_app_data/via_app_data.py

@@ -0,0 +1,140 @@
+"""Bootstrap"""
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import sys
+import traceback
+from contextlib import contextmanager
+from subprocess import CalledProcessError
+from threading import Lock, Thread
+
+from virtualenv.info import fs_supports_symlink
+from virtualenv.seed.embed.base_embed import BaseEmbed
+from virtualenv.seed.wheels import get_wheel
+from virtualenv.util.path import Path
+
+from .pip_install.copy import CopyPipInstall
+from .pip_install.symlink import SymlinkPipInstall
+
+
+class FromAppData(BaseEmbed):
+    def __init__(self, options):
+        super(FromAppData, self).__init__(options)
+        self.symlinks = options.symlink_app_data
+
+    @classmethod
+    def add_parser_arguments(cls, parser, interpreter, app_data):
+        super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data)
+        can_symlink = app_data.transient is False and fs_supports_symlink()
+        parser.add_argument(
+            "--symlink-app-data",
+            dest="symlink_app_data",
+            action="store_true" if can_symlink else "store_false",
+            help="{} symlink the python packages from the app-data folder (requires seed pip>=19.3)".format(
+                "" if can_symlink else "not supported - ",
+            ),
+            default=False,
+        )
+
+    def run(self, creator):
+        if not self.enabled:
+            return
+        with self._get_seed_wheels(creator) as name_to_whl:
+            pip_version = name_to_whl["pip"].version_tuple if "pip" in name_to_whl else None
+            installer_class = self.installer_class(pip_version)
+            exceptions = {}
+
+            def _install(name, wheel):
+                try:
+                    logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__)
+                    key = Path(installer_class.__name__) / wheel.path.stem
+                    wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key)
+                    installer = installer_class(wheel.path, creator, wheel_img)
+                    parent = self.app_data.lock / wheel_img.parent
+                    with parent.non_reentrant_lock_for_key(wheel_img.name):
+                        if not installer.has_image():
+                            installer.build_image()
+                    installer.install(creator.interpreter.version_info)
+                except Exception:  # noqa
+                    exceptions[name] = sys.exc_info()
+
+            threads = list(Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items())
+            for thread in threads:
+                thread.start()
+            for thread in threads:
+                thread.join()
+            if exceptions:
+                messages = ["failed to build image {} because:".format(", ".join(exceptions.keys()))]
+                for value in exceptions.values():
+                    exc_type, exc_value, exc_traceback = value
+                    messages.append("".join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
+                raise RuntimeError("\n".join(messages))
+
+    @contextmanager
+    def _get_seed_wheels(self, creator):
+        name_to_whl, lock, fail = {}, Lock(), {}
+
+        def _get(distribution, version):
+            for_py_version = creator.interpreter.version_release_str
+            failure, result = None, None
+            # fallback to download in case the exact version is not available
+            for download in [True] if self.download else [False, True]:
+                failure = None
+                try:
+                    result = get_wheel(
+                        distribution=distribution,
+                        version=version,
+                        for_py_version=for_py_version,
+                        search_dirs=self.extra_search_dir,
+                        download=download,
+                        app_data=self.app_data,
+                        do_periodic_update=self.periodic_update,
+                        env=self.env,
+                    )
+                    if result is not None:
+                        break
+                except Exception as exception:  # noqa
+                    logging.exception("fail")
+                    failure = exception
+            if failure:
+                if isinstance(failure, CalledProcessError):
+                    msg = "failed to download {}".format(distribution)
+                    if version is not None:
+                        msg += " version {}".format(version)
+                    msg += ", pip download exit code {}".format(failure.returncode)
+                    output = failure.output if sys.version_info < (3, 5) else (failure.output + failure.stderr)
+                    if output:
+                        msg += "\n"
+                        msg += output
+                else:
+                    msg = repr(failure)
+                logging.error(msg)
+                with lock:
+                    fail[distribution] = version
+            else:
+                with lock:
+                    name_to_whl[distribution] = result
+
+        threads = list(
+            Thread(target=_get, args=(distribution, version))
+            for distribution, version in self.distribution_to_versions().items()
+        )
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+        if fail:
+            raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys())))
+        yield name_to_whl
+
+    def installer_class(self, pip_version_tuple):
+        if self.symlinks and pip_version_tuple:
+            # symlink support requires pip 19.3+
+            if pip_version_tuple >= (19, 3):
+                return SymlinkPipInstall
+        return CopyPipInstall
+
+    def __unicode__(self):
+        base = super(FromAppData, self).__unicode__()
+        msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data)
+        return base[:-1] + msg + base[-1]