This commit is contained in:
Waylon Walker 2022-03-31 20:20:07 -05:00
commit 38355d2442
No known key found for this signature in database
GPG key ID: 66E2BF2B4190EFE4
9083 changed files with 1225834 additions and 0 deletions

View file

@ -0,0 +1,130 @@
"""Patches that are applied at runtime to the virtual environment"""
# -*- coding: utf-8 -*-
import os
import sys
VIRTUALENV_PATCH_FILE = os.path.join(__file__)
def patch_dist(dist):
"""
Distutils allows user to configure some arguments via a configuration file:
https://docs.python.org/3/install/index.html#distutils-configuration-files
Some of this arguments though don't make sense in context of the virtual environment files, let's fix them up.
"""
# we cannot allow some install config as that would get packages installed outside of the virtual environment
old_parse_config_files = dist.Distribution.parse_config_files
def parse_config_files(self, *args, **kwargs):
result = old_parse_config_files(self, *args, **kwargs)
install = self.get_option_dict("install")
if "prefix" in install: # the prefix governs where to install the libraries
install["prefix"] = VIRTUALENV_PATCH_FILE, os.path.abspath(sys.prefix)
for base in ("purelib", "platlib", "headers", "scripts", "data"):
key = "install_{}".format(base)
if key in install: # do not allow global configs to hijack venv paths
install.pop(key, None)
return result
dist.Distribution.parse_config_files = parse_config_files
# Import hook that patches some modules to ignore configuration values that break package installation in case
# of virtual environments.
_DISTUTILS_PATCH = "distutils.dist", "setuptools.dist"
if sys.version_info > (3, 4):
# https://docs.python.org/3/library/importlib.html#setting-up-an-importer
class _Finder:
"""A meta path finder that allows patching the imported distutils modules"""
fullname = None
# lock[0] is threading.Lock(), but initialized lazily to avoid importing threading very early at startup,
# because there are gevent-based applications that need to be first to import threading by themselves.
# See https://github.com/pypa/virtualenv/issues/1895 for details.
lock = []
def find_spec(self, fullname, path, target=None):
if fullname in _DISTUTILS_PATCH and self.fullname is None:
# initialize lock[0] lazily
if len(self.lock) == 0:
import threading
lock = threading.Lock()
# there is possibility that two threads T1 and T2 are simultaneously running into find_spec,
# observing .lock as empty, and further going into hereby initialization. However due to the GIL,
# list.append() operation is atomic and this way only one of the threads will "win" to put the lock
# - that every thread will use - into .lock[0].
# https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe
self.lock.append(lock)
from functools import partial
from importlib.util import find_spec
with self.lock[0]:
self.fullname = fullname
try:
spec = find_spec(fullname, path)
if spec is not None:
# https://www.python.org/dev/peps/pep-0451/#how-loading-will-work
is_new_api = hasattr(spec.loader, "exec_module")
func_name = "exec_module" if is_new_api else "load_module"
old = getattr(spec.loader, func_name)
func = self.exec_module if is_new_api else self.load_module
if old is not func:
try:
setattr(spec.loader, func_name, partial(func, old))
except AttributeError:
pass # C-Extension loaders are r/o such as zipimporter with <python 3.7
return spec
finally:
self.fullname = None
@staticmethod
def exec_module(old, module):
old(module)
if module.__name__ in _DISTUTILS_PATCH:
patch_dist(module)
@staticmethod
def load_module(old, name):
module = old(name)
if module.__name__ in _DISTUTILS_PATCH:
patch_dist(module)
return module
sys.meta_path.insert(0, _Finder())
else:
# https://www.python.org/dev/peps/pep-0302/
from imp import find_module
from pkgutil import ImpImporter, ImpLoader
class _VirtualenvImporter(object, ImpImporter):
def __init__(self, path=None):
object.__init__(self)
ImpImporter.__init__(self, path)
def find_module(self, fullname, path=None):
if fullname in _DISTUTILS_PATCH:
try:
return _VirtualenvLoader(fullname, *find_module(fullname.split(".")[-1], path))
except ImportError:
pass
return None
class _VirtualenvLoader(object, ImpLoader):
def __init__(self, fullname, file, filename, etc):
object.__init__(self)
ImpLoader.__init__(self, fullname, file, filename, etc)
def load_module(self, fullname):
module = super(_VirtualenvLoader, self).load_module(fullname)
patch_dist(module)
module.__loader__ = None # distlib fallback
return module
sys.meta_path.append(_VirtualenvImporter())

View file

@ -0,0 +1,112 @@
from __future__ import absolute_import, unicode_literals
import logging
import os
from abc import ABCMeta
from six import add_metaclass
from virtualenv.info import fs_supports_symlink
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_text
from ..creator import Creator, CreatorMeta
class ViaGlobalRefMeta(CreatorMeta):
def __init__(self):
super(ViaGlobalRefMeta, self).__init__()
self.copy_error = None
self.symlink_error = None
if not fs_supports_symlink():
self.symlink_error = "the filesystem does not supports symlink"
@property
def can_copy(self):
return not self.copy_error
@property
def can_symlink(self):
return not self.symlink_error
@add_metaclass(ABCMeta)
class ViaGlobalRefApi(Creator):
def __init__(self, options, interpreter):
super(ViaGlobalRefApi, self).__init__(options, interpreter)
self.symlinks = self._should_symlink(options)
self.enable_system_site_package = options.system_site
@staticmethod
def _should_symlink(options):
# Priority of where the option is set to follow the order: CLI, env var, file, hardcoded.
# If both set at same level prefers copy over symlink.
copies, symlinks = getattr(options, "copies", False), getattr(options, "symlinks", False)
copy_src, sym_src = options.get_source("copies"), options.get_source("symlinks")
for level in ["cli", "env var", "file", "default"]:
s_opt = symlinks if sym_src == level else None
c_opt = copies if copy_src == level else None
if s_opt is True and c_opt is True:
return False
if s_opt is True:
return True
if c_opt is True:
return False
return False # fallback to copy
@classmethod
def add_parser_arguments(cls, parser, interpreter, meta, app_data):
super(ViaGlobalRefApi, cls).add_parser_arguments(parser, interpreter, meta, app_data)
parser.add_argument(
"--system-site-packages",
default=False,
action="store_true",
dest="system_site",
help="give the virtual environment access to the system site-packages dir",
)
group = parser.add_mutually_exclusive_group()
if not meta.can_symlink and not meta.can_copy:
raise RuntimeError("neither symlink or copy method supported")
if meta.can_symlink:
group.add_argument(
"--symlinks",
default=True,
action="store_true",
dest="symlinks",
help="try to use symlinks rather than copies, when symlinks are not the default for the platform",
)
if meta.can_copy:
group.add_argument(
"--copies",
"--always-copy",
default=not meta.can_symlink,
action="store_true",
dest="copies",
help="try to use copies rather than symlinks, even when symlinks are the default for the platform",
)
def create(self):
self.install_patch()
def install_patch(self):
text = self.env_patch_text()
if text:
pth = self.purelib / "_virtualenv.pth"
logging.debug("create virtualenv import hook file %s", ensure_text(str(pth)))
pth.write_text("import _virtualenv")
dest_path = self.purelib / "_virtualenv.py"
logging.debug("create %s", ensure_text(str(dest_path)))
dest_path.write_text(text)
def env_patch_text(self):
"""Patch the distutils package to not be derailed by its configuration files"""
with self.app_data.ensure_extracted(Path(__file__).parent / "_virtualenv.py") as resolved_path:
text = resolved_path.read_text()
return text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib))))
def _args(self):
return super(ViaGlobalRefApi, self)._args() + [("global", self.enable_system_site_package)]
def set_pyenv_cfg(self):
super(ViaGlobalRefApi, self).set_pyenv_cfg()
self.pyenv_cfg["include-system-site-packages"] = "true" if self.enable_system_site_package else "false"

View file

@ -0,0 +1,17 @@
from __future__ import absolute_import, unicode_literals
from abc import ABCMeta
from six import add_metaclass
from virtualenv.create.creator import Creator
from virtualenv.create.describe import Describe
@add_metaclass(ABCMeta)
class VirtualenvBuiltin(Creator, Describe):
"""A creator that does operations itself without delegation, if we can create it we can also describe it"""
def __init__(self, options, interpreter):
Creator.__init__(self, options, interpreter)
Describe.__init__(self, self.dest, interpreter)

View file

@ -0,0 +1 @@
from __future__ import absolute_import, unicode_literals

View file

@ -0,0 +1,65 @@
from __future__ import absolute_import, unicode_literals
from abc import ABCMeta
from collections import OrderedDict
from six import add_metaclass
from virtualenv.create.describe import PosixSupports, WindowsSupports
from virtualenv.create.via_global_ref.builtin.ref import RefMust, RefWhen
from virtualenv.util.path import Path
from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@add_metaclass(ABCMeta)
class CPython(ViaGlobalRefVirtualenvBuiltin):
@classmethod
def can_describe(cls, interpreter):
return interpreter.implementation == "CPython" and super(CPython, cls).can_describe(interpreter)
@classmethod
def exe_stem(cls):
return "python"
@add_metaclass(ABCMeta)
class CPythonPosix(CPython, PosixSupports):
"""Create a CPython virtual environment on POSIX platforms"""
@classmethod
def _executables(cls, interpreter):
host_exe = Path(interpreter.system_executable)
major, minor = interpreter.version_info.major, interpreter.version_info.minor
targets = OrderedDict(
(i, None) for i in ["python", "python{}".format(major), "python{}.{}".format(major, minor), host_exe.name]
)
must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
yield host_exe, list(targets.keys()), must, RefWhen.ANY
@add_metaclass(ABCMeta)
class CPythonWindows(CPython, WindowsSupports):
@classmethod
def _executables(cls, interpreter):
# symlink of the python executables does not work reliably, copy always instead
# - https://bugs.python.org/issue42013
# - venv
host = cls.host_python(interpreter)
for path in (host.parent / n for n in {"python.exe", host.name}):
yield host, [path.name], RefMust.COPY, RefWhen.ANY
# for more info on pythonw.exe see https://stackoverflow.com/a/30313091
python_w = host.parent / "pythonw.exe"
yield python_w, [python_w.name], RefMust.COPY, RefWhen.ANY
@classmethod
def host_python(cls, interpreter):
return Path(interpreter.system_executable)
def is_mac_os_framework(interpreter):
if interpreter.platform == "darwin":
framework_var = interpreter.sysconfig_vars.get("PYTHONFRAMEWORK")
value = "Python3" if interpreter.version_info.major == 3 else "Python"
return framework_var == value
return False

View file

@ -0,0 +1,102 @@
from __future__ import absolute_import, unicode_literals
import abc
import logging
from six import add_metaclass
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.util.path import Path
from ..python2.python2 import Python2
from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework
@add_metaclass(abc.ABCMeta)
class CPython2(CPython, Python2):
"""Create a CPython version 2 virtual environment"""
@classmethod
def sources(cls, interpreter):
for src in super(CPython2, cls).sources(interpreter):
yield src
# include folder needed on Python 2 as we don't have pyenv.cfg
host_include_marker = cls.host_include_marker(interpreter)
if host_include_marker.exists():
yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
@classmethod
def needs_stdlib_py_module(cls):
return False
@classmethod
def host_include_marker(cls, interpreter):
return Path(interpreter.system_include) / "Python.h"
@property
def include(self):
# the pattern include the distribution name too at the end, remove that via the parent call
return (self.dest / self.interpreter.install_path("headers")).parent
@classmethod
def modules(cls):
return [
"os", # landmark to set sys.prefix
]
def ensure_directories(self):
dirs = super(CPython2, self).ensure_directories()
host_include_marker = self.host_include_marker(self.interpreter)
if host_include_marker.exists():
dirs.add(self.include.parent)
else:
logging.debug("no include folders as can't find include marker %s", host_include_marker)
return dirs
@add_metaclass(abc.ABCMeta)
class CPython2PosixBase(CPython2, CPythonPosix):
"""common to macOs framework builds and other posix CPython2"""
@classmethod
def sources(cls, interpreter):
for src in super(CPython2PosixBase, cls).sources(interpreter):
yield src
# check if the makefile exists and if so make it available under the virtual environment
make_file = Path(interpreter.sysconfig["makefile_filename"])
if make_file.exists() and str(make_file).startswith(interpreter.prefix):
under_prefix = make_file.relative_to(Path(interpreter.prefix))
yield PathRefToDest(make_file, dest=lambda self, s: self.dest / under_prefix)
class CPython2Posix(CPython2PosixBase):
"""CPython 2 on POSIX (excluding macOs framework builds)"""
@classmethod
def can_describe(cls, interpreter):
return is_mac_os_framework(interpreter) is False and super(CPython2Posix, cls).can_describe(interpreter)
@classmethod
def sources(cls, interpreter):
for src in super(CPython2Posix, cls).sources(interpreter):
yield src
# landmark for exec_prefix
exec_marker_file, to_path, _ = cls.from_stdlib(cls.mappings(interpreter), "lib-dynload")
yield PathRefToDest(exec_marker_file, dest=to_path)
class CPython2Windows(CPython2, CPythonWindows):
"""CPython 2 on Windows"""
@classmethod
def sources(cls, interpreter):
for src in super(CPython2Windows, cls).sources(interpreter):
yield src
py27_dll = Path(interpreter.system_executable).parent / "python27.dll"
if py27_dll.exists(): # this might be global in the Windows folder in which case it's alright to be missing
yield PathRefToDest(py27_dll, dest=cls.to_bin)
libs = Path(interpreter.system_prefix) / "libs"
if libs.exists():
yield PathRefToDest(libs, dest=lambda self, s: self.dest / s.name)

View file

@ -0,0 +1,91 @@
from __future__ import absolute_import, unicode_literals
import abc
from textwrap import dedent
from six import add_metaclass
from virtualenv.create.describe import Python3Supports
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.create.via_global_ref.store import is_store_python
from virtualenv.util.path import Path
from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework
@add_metaclass(abc.ABCMeta)
class CPython3(CPython, Python3Supports):
""" """
class CPython3Posix(CPythonPosix, CPython3):
@classmethod
def can_describe(cls, interpreter):
return is_mac_os_framework(interpreter) is False and super(CPython3Posix, cls).can_describe(interpreter)
def env_patch_text(self):
text = super(CPython3Posix, self).env_patch_text()
if self.pyvenv_launch_patch_active(self.interpreter):
text += dedent(
"""
# for https://github.com/python/cpython/pull/9516, see https://github.com/pypa/virtualenv/issues/1704
import os
if "__PYVENV_LAUNCHER__" in os.environ:
del os.environ["__PYVENV_LAUNCHER__"]
""",
)
return text
@classmethod
def pyvenv_launch_patch_active(cls, interpreter):
ver = interpreter.version_info
return interpreter.platform == "darwin" and ((3, 7, 8) > ver >= (3, 7) or (3, 8, 3) > ver >= (3, 8))
class CPython3Windows(CPythonWindows, CPython3):
""" """
@classmethod
def setup_meta(cls, interpreter):
if is_store_python(interpreter): # store python is not supported here
return None
return super(CPython3Windows, cls).setup_meta(interpreter)
@classmethod
def sources(cls, interpreter):
for src in super(CPython3Windows, cls).sources(interpreter):
yield src
if not cls.has_shim(interpreter):
for src in cls.include_dll_and_pyd(interpreter):
yield src
@classmethod
def has_shim(cls, interpreter):
return interpreter.version_info.minor >= 7 and cls.shim(interpreter) is not None
@classmethod
def shim(cls, interpreter):
shim = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe"
if shim.exists():
return shim
return None
@classmethod
def host_python(cls, interpreter):
if cls.has_shim(interpreter):
# starting with CPython 3.7 Windows ships with a venvlauncher.exe that avoids the need for dll/pyd copies
# it also means the wrapper must be copied to avoid bugs such as https://bugs.python.org/issue42013
return cls.shim(interpreter)
return super(CPython3Windows, cls).host_python(interpreter)
@classmethod
def include_dll_and_pyd(cls, interpreter):
dll_folder = Path(interpreter.system_prefix) / "DLLs"
host_exe_folder = Path(interpreter.system_executable).parent
for folder in [host_exe_folder, dll_folder]:
for file in folder.iterdir():
if file.suffix in (".pyd", ".dll"):
yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
def to_dll_and_pyd(self, src):
return self.bin_dir / src.name

View file

@ -0,0 +1,341 @@
# -*- coding: utf-8 -*-
"""The Apple Framework builds require their own customization"""
import logging
import os
import struct
import subprocess
from abc import ABCMeta, abstractmethod
from textwrap import dedent
from six import add_metaclass, text_type
from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest, RefMust
from virtualenv.info import IS_MAC_ARM64
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_text
from .common import CPython, CPythonPosix, is_mac_os_framework
from .cpython2 import CPython2PosixBase
from .cpython3 import CPython3
@add_metaclass(ABCMeta)
class CPythonmacOsFramework(CPython):
@classmethod
def can_describe(cls, interpreter):
return is_mac_os_framework(interpreter) and super(CPythonmacOsFramework, cls).can_describe(interpreter)
@classmethod
def sources(cls, interpreter):
for src in super(CPythonmacOsFramework, cls).sources(interpreter):
yield src
# add a symlink to the host python image
exe = cls.image_ref(interpreter)
ref = PathRefToDest(exe, dest=lambda self, _: self.dest / ".Python", must=RefMust.SYMLINK)
yield ref
def create(self):
super(CPythonmacOsFramework, self).create()
# change the install_name of the copied python executables
target = "@executable_path/../.Python"
current = self.current_mach_o_image_path()
for src in self._sources:
if isinstance(src, ExePathRefToDest):
if src.must == RefMust.COPY or not self.symlinks:
exes = [self.bin_dir / src.base]
if not self.symlinks:
exes.extend(self.bin_dir / a for a in src.aliases)
for exe in exes:
fix_mach_o(str(exe), current, target, self.interpreter.max_size)
@classmethod
def _executables(cls, interpreter):
for _, targets, must, when in super(CPythonmacOsFramework, cls)._executables(interpreter):
# Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
# stub executable in ${sys.prefix}/bin.
# See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
yield fixed_host_exe, targets, must, when
@abstractmethod
def current_mach_o_image_path(self):
raise NotImplementedError
@classmethod
def image_ref(cls, interpreter):
raise NotImplementedError
class CPython2macOsFramework(CPythonmacOsFramework, CPython2PosixBase):
@classmethod
def can_create(cls, interpreter):
if not IS_MAC_ARM64 and super(CPython2macOsFramework, cls).can_describe(interpreter):
return super(CPython2macOsFramework, cls).can_create(interpreter)
return False
@classmethod
def image_ref(cls, interpreter):
return Path(interpreter.prefix) / "Python"
def current_mach_o_image_path(self):
return os.path.join(self.interpreter.prefix, "Python")
@classmethod
def sources(cls, interpreter):
for src in super(CPython2macOsFramework, cls).sources(interpreter):
yield src
# landmark for exec_prefix
exec_marker_file, to_path, _ = cls.from_stdlib(cls.mappings(interpreter), "lib-dynload")
yield PathRefToDest(exec_marker_file, dest=to_path)
@property
def reload_code(self):
result = super(CPython2macOsFramework, self).reload_code
result = dedent(
"""
# the bundled site.py always adds the global site package if we're on python framework build, escape this
import sysconfig
config = sysconfig.get_config_vars()
before = config["PYTHONFRAMEWORK"]
try:
config["PYTHONFRAMEWORK"] = ""
{}
finally:
config["PYTHONFRAMEWORK"] = before
""".format(
result,
),
)
return result
class CPython2macOsArmFramework(CPython2macOsFramework, CPythonmacOsFramework, CPython2PosixBase):
@classmethod
def can_create(cls, interpreter):
if IS_MAC_ARM64 and super(CPythonmacOsFramework, cls).can_describe(interpreter):
return super(CPythonmacOsFramework, cls).can_create(interpreter)
return False
def create(self):
super(CPython2macOsFramework, self).create()
self.fix_signature()
def fix_signature(self):
"""
On Apple M1 machines (arm64 chips), rewriting the python executable invalidates its signature.
In python2 this results in a unusable python exe which just dies.
As a temporary workaround we can codesign the python exe during the creation process.
"""
exe = self.exe
try:
logging.debug("Changing signature of copied python exe %s", exe)
bak_dir = exe.parent / "bk"
# Reset the signing on Darwin since the exe has been modified.
# Note codesign fails on the original exe, it needs to be copied and moved back.
bak_dir.mkdir(parents=True, exist_ok=True)
subprocess.check_call(["cp", text_type(exe), text_type(bak_dir)])
subprocess.check_call(["mv", text_type(bak_dir / exe.name), text_type(exe)])
bak_dir.rmdir()
metadata = "--preserve-metadata=identifier,entitlements,flags,runtime"
cmd = ["codesign", "-s", "-", metadata, "-f", text_type(exe)]
logging.debug("Changing Signature: %s", cmd)
subprocess.check_call(cmd)
except Exception:
logging.fatal("Could not change MacOS code signing on copied python exe at %s", exe)
raise
class CPython3macOsFramework(CPythonmacOsFramework, CPython3, CPythonPosix):
@classmethod
def image_ref(cls, interpreter):
return Path(interpreter.prefix) / "Python3"
def current_mach_o_image_path(self):
return "@executable_path/../../../../Python3"
@property
def reload_code(self):
result = super(CPython3macOsFramework, self).reload_code
result = dedent(
"""
# the bundled site.py always adds the global site package if we're on python framework build, escape this
import sys
before = sys._framework
try:
sys._framework = None
{}
finally:
sys._framework = before
""".format(
result,
),
)
return result
def fix_mach_o(exe, current, new, max_size):
"""
https://en.wikipedia.org/wiki/Mach-O
Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries,
dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and
faster access to information in the symbol table.
Each Mach-O file is made up of one Mach-O header, followed by a series of load commands, followed by one or more
segments, each of which contains between 0 and 255 sections. Mach-O uses the REL relocation format to handle
references to symbols. When looking up symbols Mach-O uses a two-level namespace that encodes each symbol into an
'object/symbol name' pair that is then linearly searched for by first the object and then the symbol name.
The basic structurea list of variable-length "load commands" that reference pages of data elsewhere in the filewas
also used in the executable file format for Accent. The Accent file format was in turn, based on an idea from Spice
Lisp.
With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes
binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac
OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions,
does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables
(found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and
unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format.
"""
try:
logging.debug("change Mach-O for %s from %s to %s", ensure_text(exe), current, ensure_text(new))
_builtin_change_mach_o(max_size)(exe, current, new)
except Exception as e:
logging.warning("Could not call _builtin_change_mac_o: %s. " "Trying to call install_name_tool instead.", e)
try:
cmd = ["install_name_tool", "-change", current, new, exe]
subprocess.check_call(cmd)
except Exception:
logging.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed")
raise
def _builtin_change_mach_o(maxint):
MH_MAGIC = 0xFEEDFACE
MH_CIGAM = 0xCEFAEDFE
MH_MAGIC_64 = 0xFEEDFACF
MH_CIGAM_64 = 0xCFFAEDFE
FAT_MAGIC = 0xCAFEBABE
BIG_ENDIAN = ">"
LITTLE_ENDIAN = "<"
LC_LOAD_DYLIB = 0xC
class FileView(object):
"""A proxy for file-like objects that exposes a given view of a file. Modified from macholib."""
def __init__(self, file_obj, start=0, size=maxint):
if isinstance(file_obj, FileView):
self._file_obj = file_obj._file_obj
else:
self._file_obj = file_obj
self._start = start
self._end = start + size
self._pos = 0
def __repr__(self):
return "<fileview [{:d}, {:d}] {!r}>".format(self._start, self._end, self._file_obj)
def tell(self):
return self._pos
def _checkwindow(self, seek_to, op):
if not (self._start <= seek_to <= self._end):
msg = "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end)
raise IOError(msg)
def seek(self, offset, whence=0):
seek_to = offset
if whence == os.SEEK_SET:
seek_to += self._start
elif whence == os.SEEK_CUR:
seek_to += self._start + self._pos
elif whence == os.SEEK_END:
seek_to += self._end
else:
raise IOError("Invalid whence argument to seek: {!r}".format(whence))
self._checkwindow(seek_to, "seek")
self._file_obj.seek(seek_to)
self._pos = seek_to - self._start
def write(self, content):
here = self._start + self._pos
self._checkwindow(here, "write")
self._checkwindow(here + len(content), "write")
self._file_obj.seek(here, os.SEEK_SET)
self._file_obj.write(content)
self._pos += len(content)
def read(self, size=maxint):
assert size >= 0
here = self._start + self._pos
self._checkwindow(here, "read")
size = min(size, self._end - here)
self._file_obj.seek(here, os.SEEK_SET)
read_bytes = self._file_obj.read(size)
self._pos += len(read_bytes)
return read_bytes
def read_data(file, endian, num=1):
"""Read a given number of 32-bits unsigned integers from the given file with the given endianness."""
res = struct.unpack(endian + "L" * num, file.read(num * 4))
if len(res) == 1:
return res[0]
return res
def mach_o_change(at_path, what, value):
"""Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value),
provided it's shorter."""
def do_macho(file, bits, endian):
# Read Mach-O header (the magic number is assumed read by the caller)
cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6)
# 64-bits header has one more field.
if bits == 64:
read_data(file, endian)
# The header is followed by n commands
for _ in range(n_commands):
where = file.tell()
# Read command header
cmd, cmd_size = read_data(file, endian, 2)
if cmd == LC_LOAD_DYLIB:
# The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the
# beginning of the command.
name_offset = read_data(file, endian)
file.seek(where + name_offset, os.SEEK_SET)
# Read the NUL terminated string
load = file.read(cmd_size - name_offset).decode()
load = load[: load.index("\0")]
# If the string is what is being replaced, overwrite it.
if load == what:
file.seek(where + name_offset, os.SEEK_SET)
file.write(value.encode() + b"\0")
# Seek to the next command
file.seek(where + cmd_size, os.SEEK_SET)
def do_file(file, offset=0, size=maxint):
file = FileView(file, offset, size)
# Read magic number
magic = read_data(file, BIG_ENDIAN)
if magic == FAT_MAGIC:
# Fat binaries contain nfat_arch Mach-O binaries
n_fat_arch = read_data(file, BIG_ENDIAN)
for _ in range(n_fat_arch):
# Read arch header
cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5)
do_file(file, offset, size)
elif magic == MH_MAGIC:
do_macho(file, 32, BIG_ENDIAN)
elif magic == MH_CIGAM:
do_macho(file, 32, LITTLE_ENDIAN)
elif magic == MH_MAGIC_64:
do_macho(file, 64, BIG_ENDIAN)
elif magic == MH_CIGAM_64:
do_macho(file, 64, LITTLE_ENDIAN)
assert len(what) >= len(value)
with open(at_path, "r+b") as f:
do_file(f)
return mach_o_change

View file

@ -0,0 +1,51 @@
from __future__ import absolute_import, unicode_literals
import abc
from six import add_metaclass
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen
from virtualenv.util.path import Path
from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@add_metaclass(abc.ABCMeta)
class PyPy(ViaGlobalRefVirtualenvBuiltin):
@classmethod
def can_describe(cls, interpreter):
return interpreter.implementation == "PyPy" and super(PyPy, cls).can_describe(interpreter)
@classmethod
def _executables(cls, interpreter):
host = Path(interpreter.system_executable)
targets = sorted("{}{}".format(name, PyPy.suffix) for name in cls.exe_names(interpreter))
must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
yield host, targets, must, RefWhen.ANY
@classmethod
def exe_names(cls, interpreter):
return {
cls.exe_stem(),
"python",
"python{}".format(interpreter.version_info.major),
"python{}.{}".format(*interpreter.version_info),
}
@classmethod
def sources(cls, interpreter):
for src in super(PyPy, cls).sources(interpreter):
yield src
for host in cls._add_shared_libs(interpreter):
yield PathRefToDest(host, dest=lambda self, s: self.bin_dir / s.name)
@classmethod
def _add_shared_libs(cls, interpreter):
# https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
python_dir = Path(interpreter.system_executable).resolve().parent
for src in cls._shared_libs(python_dir):
yield src
@classmethod
def _shared_libs(cls, python_dir):
raise NotImplementedError

View file

@ -0,0 +1,125 @@
from __future__ import absolute_import, unicode_literals
import abc
import logging
import os
from six import add_metaclass
from virtualenv.create.describe import PosixSupports, WindowsSupports
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.util.path import Path
from ..python2.python2 import Python2
from .common import PyPy
@add_metaclass(abc.ABCMeta)
class PyPy2(PyPy, Python2):
""" """
@classmethod
def exe_stem(cls):
return "pypy"
@classmethod
def sources(cls, interpreter):
for src in super(PyPy2, cls).sources(interpreter):
yield src
# include folder needed on Python 2 as we don't have pyenv.cfg
host_include_marker = cls.host_include_marker(interpreter)
if host_include_marker.exists():
yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
@classmethod
def needs_stdlib_py_module(cls):
return True
@classmethod
def host_include_marker(cls, interpreter):
return Path(interpreter.system_include) / "PyPy.h"
@property
def include(self):
return self.dest / self.interpreter.install_path("headers")
@classmethod
def modules(cls):
# pypy2 uses some modules before the site.py loads, so we need to include these too
return super(PyPy2, cls).modules() + [
"os",
"copy_reg",
"genericpath",
"linecache",
"stat",
"UserDict",
"warnings",
]
@property
def lib_pypy(self):
return self.dest / "lib_pypy"
def ensure_directories(self):
dirs = super(PyPy2, self).ensure_directories()
dirs.add(self.lib_pypy)
host_include_marker = self.host_include_marker(self.interpreter)
if host_include_marker.exists():
dirs.add(self.include.parent)
else:
logging.debug("no include folders as can't find include marker %s", host_include_marker)
return dirs
@property
def skip_rewrite(self):
"""
PyPy2 built-in imports are handled by this path entry, don't overwrite to not disable it
see: https://github.com/pypa/virtualenv/issues/1652
"""
return 'or path.endswith("lib_pypy{}__extensions__") # PyPy2 built-in import marker'.format(os.sep)
class PyPy2Posix(PyPy2, PosixSupports):
"""PyPy 2 on POSIX"""
@classmethod
def modules(cls):
return super(PyPy2Posix, cls).modules() + ["posixpath"]
@classmethod
def _shared_libs(cls, python_dir):
return python_dir.glob("libpypy*.*")
@property
def lib(self):
return self.dest / "lib"
@classmethod
def sources(cls, interpreter):
for src in super(PyPy2Posix, cls).sources(interpreter):
yield src
host_lib = Path(interpreter.system_prefix) / "lib"
if host_lib.exists():
yield PathRefToDest(host_lib, dest=lambda self, _: self.lib)
class Pypy2Windows(PyPy2, WindowsSupports):
"""PyPy 2 on Windows"""
@classmethod
def modules(cls):
return super(Pypy2Windows, cls).modules() + ["ntpath"]
@classmethod
def _shared_libs(cls, python_dir):
# No glob in python2 PathLib
for candidate in ["libpypy-c.dll", "libffi-7.dll", "libffi-8.dll"]:
dll = python_dir / candidate
if dll.exists():
yield dll
@classmethod
def sources(cls, interpreter):
for src in super(Pypy2Windows, cls).sources(interpreter):
yield src
yield PathRefToDest(Path(interpreter.system_prefix) / "libs", dest=lambda self, s: self.dest / s.name)

View file

@ -0,0 +1,85 @@
from __future__ import absolute_import, unicode_literals
import abc
from six import add_metaclass
from virtualenv.create.describe import PosixSupports, Python3Supports, WindowsSupports
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.util.path import Path
from .common import PyPy
@add_metaclass(abc.ABCMeta)
class PyPy3(PyPy, Python3Supports):
@classmethod
def exe_stem(cls):
return "pypy3"
@classmethod
def exe_names(cls, interpreter):
return super(PyPy3, cls).exe_names(interpreter) | {"pypy"}
class PyPy3Posix(PyPy3, PosixSupports):
"""PyPy 3 on POSIX"""
@property
def stdlib(self):
"""PyPy3 respects sysconfig only for the host python, virtual envs is instead lib/pythonx.y/site-packages"""
return self.dest / "lib" / "pypy{}".format(self.interpreter.version_release_str) / "site-packages"
@classmethod
def _shared_libs(cls, python_dir):
# glob for libpypy3-c.so, libpypy3-c.dylib, libpypy3.9-c.so ...
return python_dir.glob("libpypy3*.*")
def to_lib(self, src):
return self.dest / "lib" / src.name
@classmethod
def sources(cls, interpreter):
for src in super(PyPy3Posix, cls).sources(interpreter):
yield src
# PyPy >= 3.8 supports a standard prefix installation, where older
# versions always used a portable/developent style installation.
# If this is a standard prefix installation, skip the below:
if interpreter.system_prefix == "/usr":
return
# Also copy/symlink anything under prefix/lib, which, for "portable"
# PyPy builds, includes the tk,tcl runtime and a number of shared
# objects. In distro-specific builds or on conda this should be empty
# (on PyPy3.8+ it will, like on CPython, hold the stdlib).
host_lib = Path(interpreter.system_prefix) / "lib"
stdlib = Path(interpreter.system_stdlib)
if host_lib.exists() and host_lib.is_dir():
for path in host_lib.iterdir():
if stdlib == path:
# For PyPy3.8+ the stdlib lives in lib/pypy3.8
# We need to avoid creating a symlink to it since that
# will defeat the purpose of a virtualenv
continue
yield PathRefToDest(path, dest=cls.to_lib)
class Pypy3Windows(PyPy3, WindowsSupports):
"""PyPy 3 on Windows"""
@property
def stdlib(self):
"""PyPy3 respects sysconfig only for the host python, virtual envs is instead Lib/site-packages"""
return self.dest / "Lib" / "site-packages"
@property
def bin_dir(self):
"""PyPy3 needs to fallback to pypy definition"""
return self.dest / "Scripts"
@classmethod
def _shared_libs(cls, python_dir):
# glob for libpypy*.dll and libffi*.dll
for pattern in ["libpypy*.dll", "libffi*.dll"]:
srcs = python_dir.glob(pattern)
for src in srcs:
yield src

View file

@ -0,0 +1,111 @@
from __future__ import absolute_import, unicode_literals
import abc
import json
import os
from six import add_metaclass
from virtualenv.create.describe import Python2Supports
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.info import IS_ZIPAPP
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_text
from virtualenv.util.zipapp import read as read_from_zipapp
from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
HERE = Path(os.path.abspath(__file__)).parent
@add_metaclass(abc.ABCMeta)
class Python2(ViaGlobalRefVirtualenvBuiltin, Python2Supports):
def create(self):
"""Perform operations needed to make the created environment work on Python 2"""
super(Python2, self).create()
# install a patched site-package, the default Python 2 site.py is not smart enough to understand pyvenv.cfg,
# so we inject a small shim that can do this, the location of this depends where it's on host
sys_std_plat = Path(self.interpreter.system_stdlib_platform)
site_py_in = (
self.stdlib_platform
if ((sys_std_plat / "site.py").exists() or (sys_std_plat / "site.pyc").exists())
else self.stdlib
)
site_py = site_py_in / "site.py"
custom_site = get_custom_site()
if IS_ZIPAPP:
custom_site_text = read_from_zipapp(custom_site)
else:
custom_site_text = custom_site.read_text()
expected = json.dumps([os.path.relpath(ensure_text(str(i)), ensure_text(str(site_py))) for i in self.libs])
custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)
reload_code = os.linesep.join(" {}".format(i) for i in self.reload_code.splitlines()).lstrip()
custom_site_text = custom_site_text.replace("# ___RELOAD_CODE___", reload_code)
skip_rewrite = os.linesep.join(" {}".format(i) for i in self.skip_rewrite.splitlines()).lstrip()
custom_site_text = custom_site_text.replace("# ___SKIP_REWRITE____", skip_rewrite)
site_py.write_text(custom_site_text)
@property
def reload_code(self):
return 'reload(sys.modules["site"]) # noqa # call system site.py to setup import libraries'
@property
def skip_rewrite(self):
return ""
@classmethod
def sources(cls, interpreter):
for src in super(Python2, cls).sources(interpreter):
yield src
# install files needed to run site.py, either from stdlib or stdlib_platform, at least pyc, but both if exists
# if neither exists return the module file to trigger failure
mappings, needs_py_module = (
cls.mappings(interpreter),
cls.needs_stdlib_py_module(),
)
for req in cls.modules():
module_file, to_module, module_exists = cls.from_stdlib(mappings, "{}.py".format(req))
compiled_file, to_compiled, compiled_exists = cls.from_stdlib(mappings, "{}.pyc".format(req))
if needs_py_module or module_exists or not compiled_exists:
yield PathRefToDest(module_file, dest=to_module)
if compiled_exists:
yield PathRefToDest(compiled_file, dest=to_compiled)
@staticmethod
def from_stdlib(mappings, name):
for from_std, to_std in mappings:
src = from_std / name
if src.exists():
return src, to_std, True
# if not exists, fallback to first in list
return mappings[0][0] / name, mappings[0][1], False
@classmethod
def mappings(cls, interpreter):
mappings = [(Path(interpreter.system_stdlib_platform), cls.to_stdlib_platform)]
if interpreter.system_stdlib_platform != interpreter.system_stdlib:
mappings.append((Path(interpreter.system_stdlib), cls.to_stdlib))
return mappings
def to_stdlib(self, src):
return self.stdlib / src.name
def to_stdlib_platform(self, src):
return self.stdlib_platform / src.name
@classmethod
def needs_stdlib_py_module(cls):
raise NotImplementedError
@classmethod
def modules(cls):
return []
def get_custom_site():
return HERE / "site.py"

View file

@ -0,0 +1,190 @@
# -*- coding: utf-8 -*-
"""
A simple shim module to fix up things on Python 2 only.
Note: until we setup correctly the paths we can only import built-ins.
"""
import sys
def main():
"""Patch what needed, and invoke the original site.py"""
here = __file__ # the distutils.install patterns will be injected relative to this site.py, save it here
config = read_pyvenv()
sys.real_prefix = sys.base_prefix = config["base-prefix"]
sys.base_exec_prefix = config["base-exec-prefix"]
sys.base_executable = config["base-executable"]
global_site_package_enabled = config.get("include-system-site-packages", False) == "true"
rewrite_standard_library_sys_path()
disable_user_site_package()
load_host_site(here)
if global_site_package_enabled:
add_global_site_package()
rewrite_getsitepackages(here)
def load_host_site(here):
"""trigger reload of site.py - now it will use the standard library instance that will take care of init"""
# we have a duality here, we generate the platform and pure library path based on what distutils.install specifies
# because this is what pip will be using; the host site.py though may contain it's own pattern for where the
# platform and pure library paths should exist
# notably on Ubuntu there's a patch for getsitepackages to point to
# - prefix + local/lib/pythonx.y/dist-packages
# - prefix + lib/pythonx.y/dist-packages
# while distutils.install.cmd still points both of these to
# - prefix + lib/python2.7/site-packages
# to facilitate when the two match, or not we first reload the site.py, now triggering the import of host site.py,
# as this will ensure that initialization code within host site.py runs
# ___RELOAD_CODE___
# and then if the distutils site packages are not on the sys.path we add them via add_site_dir; note we must add
# them by invoking add_site_dir to trigger the processing of pth files
add_site_dir = sys.modules["site"].addsitedir
for path in get_site_packages_dirs(here):
add_site_dir(path)
def get_site_packages_dirs(here):
import json
import os
site_packages = r"""
___EXPECTED_SITE_PACKAGES___
"""
for path in json.loads(site_packages):
yield os.path.abspath(os.path.join(here, path.encode("utf-8")))
sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version
def read_pyvenv():
"""read pyvenv.cfg"""
config_file = "{}{}pyvenv.cfg".format(sys.prefix, sep)
with open(config_file) as file_handler:
lines = file_handler.readlines()
config = {}
for line in lines:
try:
split_at = line.index("=")
except ValueError:
continue # ignore bad/empty lines
else:
config[line[:split_at].strip()] = line[split_at + 1 :].strip()
return config
def rewrite_standard_library_sys_path():
"""Once this site file is loaded the standard library paths have already been set, fix them up"""
exe, prefix, exec_prefix = get_exe_prefixes(base=False)
base_exe, base_prefix, base_exec = get_exe_prefixes(base=True)
exe_dir = exe[: exe.rfind(sep)]
for at, path in enumerate(sys.path):
path = abs_path(path) # replace old sys prefix path starts with new
skip_rewrite = path == exe_dir # don't fix the current executable location, notably on Windows this gets added
skip_rewrite = skip_rewrite # ___SKIP_REWRITE____
if not skip_rewrite:
sys.path[at] = map_path(path, base_exe, exe_dir, exec_prefix, base_prefix, prefix, base_exec)
# the rewrite above may have changed elements from PYTHONPATH, revert these if on
if sys.flags.ignore_environment:
return
import os
python_paths = []
if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]:
for path in os.environ["PYTHONPATH"].split(os.pathsep):
if path not in python_paths:
python_paths.append(path)
sys.path[: len(python_paths)] = python_paths
def get_exe_prefixes(base=False):
return tuple(abs_path(getattr(sys, ("base_" if base else "") + i)) for i in ("executable", "prefix", "exec_prefix"))
def abs_path(value):
values, keep = value.split(sep), []
at = len(values) - 1
while at >= 0:
if values[at] == "..":
at -= 1
else:
keep.append(values[at])
at -= 1
return sep.join(keep[::-1])
def map_path(path, base_executable, exe_dir, exec_prefix, base_prefix, prefix, base_exec_prefix):
if path_starts_with(path, exe_dir):
# content inside the exe folder needs to remap to original executables folder
orig_exe_folder = base_executable[: base_executable.rfind(sep)]
return "{}{}".format(orig_exe_folder, path[len(exe_dir) :])
elif path_starts_with(path, prefix):
return "{}{}".format(base_prefix, path[len(prefix) :])
elif path_starts_with(path, exec_prefix):
return "{}{}".format(base_exec_prefix, path[len(exec_prefix) :])
return path
def path_starts_with(directory, value):
return directory.startswith(value if value[-1] == sep else value + sep)
def disable_user_site_package():
"""Flip the switch on enable user site package"""
# sys.flags is a c-extension type, so we cannot monkeypatch it, replace it with a python class to flip it
sys.original_flags = sys.flags
class Flags(object):
def __init__(self):
self.__dict__ = {key: getattr(sys.flags, key) for key in dir(sys.flags) if not key.startswith("_")}
sys.flags = Flags()
sys.flags.no_user_site = 1
def add_global_site_package():
"""add the global site package"""
import site
# add user site package
sys.flags = sys.original_flags # restore original
site.ENABLE_USER_SITE = None # reset user site check
# add the global site package to the path - use new prefix and delegate to site.py
orig_prefixes = None
try:
orig_prefixes = site.PREFIXES
site.PREFIXES = [sys.base_prefix, sys.base_exec_prefix]
site.main()
finally:
site.PREFIXES = orig_prefixes + site.PREFIXES
# Debian and it's derivatives patch this function. We undo the damage
def rewrite_getsitepackages(here):
site = sys.modules["site"]
site_package_dirs = get_site_packages_dirs(here)
orig_getsitepackages = site.getsitepackages
def getsitepackages():
sitepackages = orig_getsitepackages()
if sys.prefix not in site.PREFIXES or sys.exec_prefix not in site.PREFIXES:
# Someone messed with the prefixes, so we stop patching
return sitepackages
for path in site_package_dirs:
if path not in sitepackages:
sitepackages.insert(0, path)
return sitepackages
site.getsitepackages = getsitepackages
main()

View file

@ -0,0 +1,172 @@
"""
Virtual environments in the traditional sense are built as reference to the host python. This file allows declarative
references to elements on the file system, allowing our system to automatically detect what modes it can support given
the constraints: e.g. can the file system symlink, can the files be read, executed, etc.
"""
from __future__ import absolute_import, unicode_literals
import os
from abc import ABCMeta, abstractmethod
from collections import OrderedDict
from stat import S_IXGRP, S_IXOTH, S_IXUSR
from six import add_metaclass
from virtualenv.info import fs_is_case_sensitive, fs_supports_symlink
from virtualenv.util.path import copy, make_exe, symlink
from virtualenv.util.six import ensure_text
class RefMust(object):
NA = "NA"
COPY = "copy"
SYMLINK = "symlink"
class RefWhen(object):
ANY = "ANY"
COPY = "copy"
SYMLINK = "symlink"
@add_metaclass(ABCMeta)
class PathRef(object):
"""Base class that checks if a file reference can be symlink/copied"""
FS_SUPPORTS_SYMLINK = fs_supports_symlink()
FS_CASE_SENSITIVE = fs_is_case_sensitive()
def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY):
self.must = must
self.when = when
self.src = src
try:
self.exists = src.exists()
except OSError:
self.exists = False
self._can_read = None if self.exists else False
self._can_copy = None if self.exists else False
self._can_symlink = None if self.exists else False
def __repr__(self):
return "{}(src={})".format(self.__class__.__name__, self.src)
@property
def can_read(self):
if self._can_read is None:
if self.src.is_file():
try:
with self.src.open("rb"):
self._can_read = True
except OSError:
self._can_read = False
else:
self._can_read = os.access(ensure_text(str(self.src)), os.R_OK)
return self._can_read
@property
def can_copy(self):
if self._can_copy is None:
if self.must == RefMust.SYMLINK:
self._can_copy = self.can_symlink
else:
self._can_copy = self.can_read
return self._can_copy
@property
def can_symlink(self):
if self._can_symlink is None:
if self.must == RefMust.COPY:
self._can_symlink = self.can_copy
else:
self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read
return self._can_symlink
@abstractmethod
def run(self, creator, symlinks):
raise NotImplementedError
def method(self, symlinks):
if self.must == RefMust.SYMLINK:
return symlink
if self.must == RefMust.COPY:
return copy
return symlink if symlinks else copy
@add_metaclass(ABCMeta)
class ExePathRef(PathRef):
"""Base class that checks if a executable can be references via symlink/copy"""
def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY):
super(ExePathRef, self).__init__(src, must, when)
self._can_run = None
@property
def can_symlink(self):
if self.FS_SUPPORTS_SYMLINK:
return self.can_run
return False
@property
def can_run(self):
if self._can_run is None:
mode = self.src.stat().st_mode
for key in [S_IXUSR, S_IXGRP, S_IXOTH]:
if mode & key:
self._can_run = True
break
else:
self._can_run = False
return self._can_run
class PathRefToDest(PathRef):
"""Link a path on the file system"""
def __init__(self, src, dest, must=RefMust.NA, when=RefWhen.ANY):
super(PathRefToDest, self).__init__(src, must, when)
self.dest = dest
def run(self, creator, symlinks):
dest = self.dest(creator, self.src)
method = self.method(symlinks)
dest_iterable = dest if isinstance(dest, list) else (dest,)
if not dest.parent.exists():
dest.parent.mkdir(parents=True, exist_ok=True)
for dst in dest_iterable:
method(self.src, dst)
class ExePathRefToDest(PathRefToDest, ExePathRef):
"""Link a exe path on the file system"""
def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY):
ExePathRef.__init__(self, src, must, when)
PathRefToDest.__init__(self, src, dest, must, when)
if not self.FS_CASE_SENSITIVE:
targets = list(OrderedDict((i.lower(), None) for i in targets).keys())
self.base = targets[0]
self.aliases = targets[1:]
self.dest = dest
def run(self, creator, symlinks):
bin_dir = self.dest(creator, self.src).parent
dest = bin_dir / self.base
method = self.method(symlinks)
method(self.src, dest)
if not symlinks:
make_exe(dest)
for extra in self.aliases:
link_file = bin_dir / extra
if link_file.exists():
link_file.unlink()
if symlinks:
link_file.symlink_to(self.base)
else:
copy(self.src, link_file)
if not symlinks:
make_exe(link_file)
def __repr__(self):
return "{}(src={}, alias={})".format(self.__class__.__name__, self.src, self.aliases)

View file

@ -0,0 +1,114 @@
from __future__ import absolute_import, unicode_literals
from abc import ABCMeta
from six import add_metaclass
from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, RefMust, RefWhen
from virtualenv.util.path import ensure_dir
from ..api import ViaGlobalRefApi, ViaGlobalRefMeta
from .builtin_way import VirtualenvBuiltin
class BuiltinViaGlobalRefMeta(ViaGlobalRefMeta):
def __init__(self):
super(BuiltinViaGlobalRefMeta, self).__init__()
self.sources = []
@add_metaclass(ABCMeta)
class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin):
def __init__(self, options, interpreter):
super(ViaGlobalRefVirtualenvBuiltin, self).__init__(options, interpreter)
self._sources = getattr(options.meta, "sources", None) # if we're created as a describer this might be missing
@classmethod
def can_create(cls, interpreter):
"""By default all built-in methods assume that if we can describe it we can create it"""
# first we must be able to describe it
if not cls.can_describe(interpreter):
return None
meta = cls.setup_meta(interpreter)
if meta is not None and meta:
cls._sources_can_be_applied(interpreter, meta)
return meta
@classmethod
def _sources_can_be_applied(cls, interpreter, meta):
for src in cls.sources(interpreter):
if src.exists:
if meta.can_copy and not src.can_copy:
meta.copy_error = "cannot copy {}".format(src)
if meta.can_symlink and not src.can_symlink:
meta.symlink_error = "cannot symlink {}".format(src)
else:
msg = "missing required file {}".format(src)
if src.when == RefMust.NA:
meta.error = msg
elif src.when == RefMust.COPY:
meta.copy_error = msg
elif src.when == RefMust.SYMLINK:
meta.symlink_error = msg
if not meta.can_copy and not meta.can_symlink:
meta.error = "neither copy or symlink supported, copy: {} symlink: {}".format(
meta.copy_error,
meta.symlink_error,
)
if meta.error:
break
meta.sources.append(src)
@classmethod
def setup_meta(cls, interpreter):
return BuiltinViaGlobalRefMeta()
@classmethod
def sources(cls, interpreter):
for host_exe, targets, must, when in cls._executables(interpreter):
yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must=must, when=when)
def to_bin(self, src):
return self.bin_dir / src.name
@classmethod
def _executables(cls, interpreter):
raise NotImplementedError
def create(self):
dirs = self.ensure_directories()
for directory in list(dirs):
if any(i for i in dirs if i is not directory and directory.parts == i.parts[: len(directory.parts)]):
dirs.remove(directory)
for directory in sorted(dirs):
ensure_dir(directory)
self.set_pyenv_cfg()
self.pyenv_cfg.write()
true_system_site = self.enable_system_site_package
try:
self.enable_system_site_package = False
for src in self._sources:
if (
src.when == RefWhen.ANY
or (src.when == RefWhen.SYMLINK and self.symlinks is True)
or (src.when == RefWhen.COPY and self.symlinks is False)
):
src.run(self, self.symlinks)
finally:
if true_system_site != self.enable_system_site_package:
self.enable_system_site_package = true_system_site
super(ViaGlobalRefVirtualenvBuiltin, self).create()
def ensure_directories(self):
return {self.dest, self.bin_dir, self.script_dir, self.stdlib} | set(self.libs)
def set_pyenv_cfg(self):
"""
We directly inject the base prefix and base exec prefix to avoid site.py needing to discover these
from home (which usually is done within the interpreter itself)
"""
super(ViaGlobalRefVirtualenvBuiltin, self).set_pyenv_cfg()
self.pyenv_cfg["base-prefix"] = self.interpreter.system_prefix
self.pyenv_cfg["base-exec-prefix"] = self.interpreter.system_exec_prefix
self.pyenv_cfg["base-executable"] = self.interpreter.system_executable

View file

@ -0,0 +1,26 @@
from __future__ import absolute_import, unicode_literals
from virtualenv.util.path import Path
def handle_store_python(meta, interpreter):
if is_store_python(interpreter):
meta.symlink_error = "Windows Store Python does not support virtual environments via symlink"
return meta
def is_store_python(interpreter):
parts = Path(interpreter.system_executable).parts
return (
len(parts) > 4
and parts[-4] == "Microsoft"
and parts[-3] == "WindowsApps"
and parts[-2].startswith("PythonSoftwareFoundation.Python.3.")
and parts[-1].startswith("python")
)
__all__ = (
"handle_store_python",
"is_store_python",
)

View file

@ -0,0 +1,83 @@
from __future__ import absolute_import, unicode_literals
import logging
from copy import copy
from virtualenv.create.via_global_ref.store import handle_store_python
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.util.error import ProcessCallFailed
from virtualenv.util.path import ensure_dir
from virtualenv.util.subprocess import run_cmd
from .api import ViaGlobalRefApi, ViaGlobalRefMeta
class Venv(ViaGlobalRefApi):
def __init__(self, options, interpreter):
self.describe = options.describe
super(Venv, self).__init__(options, interpreter)
self.can_be_inline = (
interpreter is PythonInfo.current() and interpreter.executable == interpreter.system_executable
)
self._context = None
def _args(self):
return super(Venv, self)._args() + ([("describe", self.describe.__class__.__name__)] if self.describe else [])
@classmethod
def can_create(cls, interpreter):
if interpreter.has_venv:
meta = ViaGlobalRefMeta()
if interpreter.platform == "win32" and interpreter.version_info.major == 3:
meta = handle_store_python(meta, interpreter)
return meta
return None
def create(self):
if self.can_be_inline:
self.create_inline()
else:
self.create_via_sub_process()
for lib in self.libs:
ensure_dir(lib)
super(Venv, self).create()
def create_inline(self):
from venv import EnvBuilder
builder = EnvBuilder(
system_site_packages=self.enable_system_site_package,
clear=False,
symlinks=self.symlinks,
with_pip=False,
)
builder.create(str(self.dest))
def create_via_sub_process(self):
cmd = self.get_host_create_cmd()
logging.info("using host built-in venv to create via %s", " ".join(cmd))
code, out, err = run_cmd(cmd)
if code != 0:
raise ProcessCallFailed(code, out, err, cmd)
def get_host_create_cmd(self):
cmd = [self.interpreter.system_executable, "-m", "venv", "--without-pip"]
if self.enable_system_site_package:
cmd.append("--system-site-packages")
cmd.append("--symlinks" if self.symlinks else "--copies")
cmd.append(str(self.dest))
return cmd
def set_pyenv_cfg(self):
# prefer venv options over ours, but keep our extra
venv_content = copy(self.pyenv_cfg.refresh())
super(Venv, self).set_pyenv_cfg()
self.pyenv_cfg.update(venv_content)
def __getattribute__(self, item):
describe = object.__getattribute__(self, "describe")
if describe is not None and hasattr(describe, item):
element = getattr(describe, item)
if not callable(element) or item in ("script",):
return element
return object.__getattribute__(self, item)