This commit is contained in:
Waylon Walker 2022-03-31 20:20:07 -05:00
commit 38355d2442
No known key found for this signature in database
GPG key ID: 66E2BF2B4190EFE4
9083 changed files with 1225834 additions and 0 deletions

View file

@ -0,0 +1 @@
from __future__ import absolute_import, unicode_literals

View file

@ -0,0 +1,186 @@
from __future__ import absolute_import, unicode_literals
import logging
import os
import sys
from virtualenv.info import IS_WIN
from virtualenv.util.six import ensure_str, ensure_text
from .discover import Discover
from .py_info import PythonInfo
from .py_spec import PythonSpec
class Builtin(Discover):
def __init__(self, options):
super(Builtin, self).__init__(options)
self.python_spec = options.python if options.python else [sys.executable]
self.app_data = options.app_data
self.try_first_with = options.try_first_with
@classmethod
def add_parser_arguments(cls, parser):
parser.add_argument(
"-p",
"--python",
dest="python",
metavar="py",
type=str,
action="append",
default=[],
help="interpreter based on what to create environment (path/identifier) "
"- by default use the interpreter where the tool is installed - first found wins",
)
parser.add_argument(
"--try-first-with",
dest="try_first_with",
metavar="py_exe",
type=str,
action="append",
default=[],
help="try first these interpreters before starting the discovery",
)
def run(self):
for python_spec in self.python_spec:
result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env)
if result is not None:
return result
return None
def __repr__(self):
return ensure_str(self.__unicode__())
def __unicode__(self):
spec = self.python_spec[0] if len(self.python_spec) == 1 else self.python_spec
return "{} discover of python_spec={!r}".format(self.__class__.__name__, spec)
def get_interpreter(key, try_first_with, app_data=None, env=None):
spec = PythonSpec.from_string_spec(key)
logging.info("find interpreter for spec %r", spec)
proposed_paths = set()
env = os.environ if env is None else env
for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env):
key = interpreter.system_executable, impl_must_match
if key in proposed_paths:
continue
logging.info("proposed %s", interpreter)
if interpreter.satisfies(spec, impl_must_match):
logging.debug("accepted %s", interpreter)
return interpreter
proposed_paths.add(key)
def propose_interpreters(spec, try_first_with, app_data, env=None):
# 0. try with first
env = os.environ if env is None else env
for py_exe in try_first_with:
path = os.path.abspath(py_exe)
try:
os.lstat(path) # Windows Store Python does not work with os.path.exists, but does for os.lstat
except OSError:
pass
else:
yield PythonInfo.from_exe(os.path.abspath(path), app_data, env=env), True
# 1. if it's a path and exists
if spec.path is not None:
try:
os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat
except OSError:
if spec.is_abs:
raise
else:
yield PythonInfo.from_exe(os.path.abspath(spec.path), app_data, env=env), True
if spec.is_abs:
return
else:
# 2. otherwise try with the current
yield PythonInfo.current_system(app_data), True
# 3. otherwise fallback to platform default logic
if IS_WIN:
from .windows import propose_interpreters
for interpreter in propose_interpreters(spec, app_data, env):
yield interpreter, True
# finally just find on path, the path order matters (as the candidates are less easy to control by end user)
paths = get_paths(env)
tested_exes = set()
for pos, path in enumerate(paths):
path = ensure_text(path)
logging.debug(LazyPathDump(pos, path, env))
for candidate, match in possible_specs(spec):
found = check_path(candidate, path)
if found is not None:
exe = os.path.abspath(found)
if exe not in tested_exes:
tested_exes.add(exe)
interpreter = PathPythonInfo.from_exe(exe, app_data, raise_on_error=False, env=env)
if interpreter is not None:
yield interpreter, match
def get_paths(env):
path = env.get(str("PATH"), None)
if path is None:
try:
path = os.confstr("CS_PATH")
except (AttributeError, ValueError):
path = os.defpath
if not path:
paths = []
else:
paths = [p for p in path.split(os.pathsep) if os.path.exists(p)]
return paths
class LazyPathDump(object):
def __init__(self, pos, path, env):
self.pos = pos
self.path = path
self.env = env
def __repr__(self):
return ensure_str(self.__unicode__())
def __unicode__(self):
content = "discover PATH[{}]={}".format(self.pos, self.path)
if self.env.get(str("_VIRTUALENV_DEBUG")): # this is the over the board debug
content += " with =>"
for file_name in os.listdir(self.path):
try:
file_path = os.path.join(self.path, file_name)
if os.path.isdir(file_path) or not os.access(file_path, os.X_OK):
continue
except OSError:
pass
content += " "
content += file_name
return content
def check_path(candidate, path):
_, ext = os.path.splitext(candidate)
if sys.platform == "win32" and ext != ".exe":
candidate = candidate + ".exe"
if os.path.isfile(candidate):
return candidate
candidate = os.path.join(path, candidate)
if os.path.isfile(candidate):
return candidate
return None
def possible_specs(spec):
# 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts
yield spec.str_spec, False
# 5. or from the spec we can deduce a name on path that matches
for exe, match in spec.generate_names():
yield exe, match
class PathPythonInfo(PythonInfo):
""" """

View file

@ -0,0 +1,152 @@
"""
We acquire the python information by running an interrogation script via subprocess trigger. This operation is not
cheap, especially not on Windows. To not have to pay this hefty cost every time we apply multiple levels of
caching.
"""
from __future__ import absolute_import, unicode_literals
import logging
import os
import pipes
import sys
from collections import OrderedDict
from virtualenv.app_data import AppDataDisabled
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.info import PY2
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_text
from virtualenv.util.subprocess import Popen, subprocess
_CACHE = OrderedDict()
_CACHE[Path(sys.executable)] = PythonInfo()
def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False):
env = os.environ if env is None else env
result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache)
if isinstance(result, Exception):
if raise_on_error:
raise result
else:
logging.info("%s", str(result))
result = None
return result
def _get_from_cache(cls, app_data, exe, env, ignore_cache=True):
# note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a
# pyenv.cfg somewhere alongside on python3.5+
exe_path = Path(exe)
if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache
result = _CACHE[exe_path]
else: # otherwise go through the app data cache
py_info = _get_via_file_cache(cls, app_data, exe_path, exe, env)
result = _CACHE[exe_path] = py_info
# independent if it was from the file or in-memory cache fix the original executable location
if isinstance(result, PythonInfo):
result.executable = exe
return result
def _get_via_file_cache(cls, app_data, path, exe, env):
path_text = ensure_text(str(path))
try:
path_modified = path.stat().st_mtime
except OSError:
path_modified = -1
if app_data is None:
app_data = AppDataDisabled()
py_info, py_info_store = None, app_data.py_info(path)
with py_info_store.locked():
if py_info_store.exists(): # if exists and matches load
data = py_info_store.read()
of_path, of_st_mtime, of_content = data["path"], data["st_mtime"], data["content"]
if of_path == path_text and of_st_mtime == path_modified:
py_info = cls._from_dict({k: v for k, v in of_content.items()})
sys_exe = py_info.system_executable
if sys_exe is not None and not os.path.exists(sys_exe):
py_info_store.remove()
py_info = None
else:
py_info_store.remove()
if py_info is None: # if not loaded run and save
failure, py_info = _run_subprocess(cls, exe, app_data, env)
if failure is None:
data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()}
py_info_store.write(data)
else:
py_info = failure
return py_info
def _run_subprocess(cls, exe, app_data, env):
py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py"
with app_data.ensure_extracted(py_info_script) as py_info_script:
cmd = [exe, str(py_info_script)]
# prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490
env = env.copy()
env.pop("__PYVENV_LAUNCHER__", None)
logging.debug("get interpreter info via cmd: %s", LogCmd(cmd))
try:
process = Popen(
cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
)
out, err = process.communicate()
code = process.returncode
except OSError as os_error:
out, err, code = "", os_error.strerror, os_error.errno
result, failure = None, None
if code == 0:
result = cls._from_json(out)
result.executable = exe # keep original executable as this may contain initialization code
else:
msg = "failed to query {} with code {}{}{}".format(
exe,
code,
" out: {!r}".format(out) if out else "",
" err: {!r}".format(err) if err else "",
)
failure = RuntimeError(msg)
return failure, result
class LogCmd(object):
def __init__(self, cmd, env=None):
self.cmd = cmd
self.env = env
def __repr__(self):
def e(v):
return v.decode("utf-8") if isinstance(v, bytes) else v
cmd_repr = e(" ").join(pipes.quote(e(c)) for c in self.cmd)
if self.env is not None:
cmd_repr += e(" env of {!r}").format(self.env)
if PY2:
return cmd_repr.encode("utf-8")
return cmd_repr
def __unicode__(self):
raw = repr(self)
if PY2:
return raw.decode("utf-8")
return raw
def clear(app_data):
app_data.py_info_clear()
_CACHE.clear()
___all___ = (
"from_exe",
"clear",
"LogCmd",
)

View file

@ -0,0 +1,47 @@
from __future__ import absolute_import, unicode_literals
from abc import ABCMeta, abstractmethod
from six import add_metaclass
@add_metaclass(ABCMeta)
class Discover(object):
"""Discover and provide the requested Python interpreter"""
@classmethod
def add_parser_arguments(cls, parser):
"""Add CLI arguments for this discovery mechanisms.
:param parser: the CLI parser
"""
raise NotImplementedError
# noinspection PyUnusedLocal
def __init__(self, options):
"""Create a new discovery mechanism.
:param options: the parsed options as defined within :meth:`add_parser_arguments`
"""
self._has_run = False
self._interpreter = None
self._env = options.env
@abstractmethod
def run(self):
"""Discovers an interpreter.
:return: the interpreter ready to use for virtual environment creation
"""
raise NotImplementedError
@property
def interpreter(self):
"""
:return: the interpreter as returned by :meth:`run`, cached
"""
if self._has_run is False:
self._interpreter = self.run()
self._has_run = True
return self._interpreter

View file

@ -0,0 +1,523 @@
"""
The PythonInfo contains information about a concrete instance of a Python interpreter
Note: this file is also used to query target interpreters, so can only use standard library methods
"""
from __future__ import absolute_import, print_function
import json
import logging
import os
import platform
import re
import sys
import sysconfig
import warnings
from collections import OrderedDict, namedtuple
from string import digits
VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"])
def _get_path_extensions():
return list(OrderedDict.fromkeys([""] + os.environ.get("PATHEXT", "").lower().split(os.pathsep)))
EXTENSIONS = _get_path_extensions()
_CONF_VAR_RE = re.compile(r"\{\w+\}")
class PythonInfo(object):
"""Contains information for a Python interpreter"""
def __init__(self):
def u(v):
return v.decode("utf-8") if isinstance(v, bytes) else v
def abs_path(v):
return None if v is None else os.path.abspath(v) # unroll relative elements from path (e.g. ..)
# qualifies the python
self.platform = u(sys.platform)
self.implementation = u(platform.python_implementation())
if self.implementation == "PyPy":
self.pypy_version_info = tuple(u(i) for i in sys.pypy_version_info)
# this is a tuple in earlier, struct later, unify to our own named tuple
self.version_info = VersionInfo(*list(u(i) for i in sys.version_info))
self.architecture = 64 if sys.maxsize > 2**32 else 32
self.version = u(sys.version)
self.os = u(os.name)
# information about the prefix - determines python home
self.prefix = u(abs_path(getattr(sys, "prefix", None))) # prefix we think
self.base_prefix = u(abs_path(getattr(sys, "base_prefix", None))) # venv
self.real_prefix = u(abs_path(getattr(sys, "real_prefix", None))) # old virtualenv
# information about the exec prefix - dynamic stdlib modules
self.base_exec_prefix = u(abs_path(getattr(sys, "base_exec_prefix", None)))
self.exec_prefix = u(abs_path(getattr(sys, "exec_prefix", None)))
self.executable = u(abs_path(sys.executable)) # the executable we were invoked via
self.original_executable = u(abs_path(self.executable)) # the executable as known by the interpreter
self.system_executable = self._fast_get_system_executable() # the executable we are based of (if available)
try:
__import__("venv")
has = True
except ImportError:
has = False
self.has_venv = has
self.path = [u(i) for i in sys.path]
self.file_system_encoding = u(sys.getfilesystemencoding())
self.stdout_encoding = u(getattr(sys.stdout, "encoding", None))
if "venv" in sysconfig.get_scheme_names():
self.sysconfig_scheme = "venv"
self.sysconfig_paths = {
u(i): u(sysconfig.get_path(i, expand=False, scheme="venv")) for i in sysconfig.get_path_names()
}
# we cannot use distutils at all if "venv" exists, distutils don't know it
self.distutils_install = {}
else:
self.sysconfig_scheme = None
self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()}
self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()}
# https://bugs.python.org/issue22199
makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None))
self.sysconfig = {
u(k): u(v)
for k, v in [
# a list of content to store from sysconfig
("makefile_filename", makefile()),
]
if k is not None
}
config_var_keys = set()
for element in self.sysconfig_paths.values():
for k in _CONF_VAR_RE.findall(element):
config_var_keys.add(u(k[1:-1]))
config_var_keys.add("PYTHONFRAMEWORK")
self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i) or "") for i in config_var_keys}
if self.implementation == "PyPy" and sys.version_info.major == 2:
self.sysconfig_vars["implementation_lower"] = "python"
confs = {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}
self.system_stdlib = self.sysconfig_path("stdlib", confs)
self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs)
self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None))
self._creators = None
def _fast_get_system_executable(self):
"""Try to get the system executable by just looking at properties"""
if self.real_prefix or (
self.base_prefix is not None and self.base_prefix != self.prefix
): # if this is a virtual environment
if self.real_prefix is None:
base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us
if base_executable is not None: # use the saved system executable if present
if sys.executable != base_executable: # we know we're in a virtual environment, cannot be us
return base_executable
return None # in this case we just can't tell easily without poking around FS and calling them, bail
# if we're not in a virtual environment, this is already a system python, so return the original executable
# note we must choose the original and not the pure executable as shim scripts might throw us off
return self.original_executable
def install_path(self, key):
result = self.distutils_install.get(key)
if result is None: # use sysconfig if sysconfig_scheme is set or distutils is unavailable
# set prefixes to empty => result is relative from cwd
prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix
config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()}
result = self.sysconfig_path(key, config_var=config_var).lstrip(os.sep)
return result
@staticmethod
def _distutils_install():
# use distutils primarily because that's what pip does
# https://github.com/pypa/pip/blob/main/src/pip/_internal/locations.py#L95
# note here we don't import Distribution directly to allow setuptools to patch it
with warnings.catch_warnings(): # disable warning for PEP-632
warnings.simplefilter("ignore")
try:
from distutils import dist
from distutils.command.install import SCHEME_KEYS
except ImportError: # if removed or not installed ignore
return {}
d = dist.Distribution({"script_args": "--no-user-cfg"}) # conf files not parsed so they do not hijack paths
if hasattr(sys, "_framework"):
sys._framework = None # disable macOS static paths for framework
with warnings.catch_warnings(): # disable warning for PEP-632
warnings.simplefilter("ignore")
i = d.get_command_obj("install", create=True)
i.prefix = os.sep # paths generated are relative to prefix that contains the path sep, this makes it relative
i.finalize_options()
result = {key: (getattr(i, "install_{}".format(key))[1:]).lstrip(os.sep) for key in SCHEME_KEYS}
return result
@property
def version_str(self):
return ".".join(str(i) for i in self.version_info[0:3])
@property
def version_release_str(self):
return ".".join(str(i) for i in self.version_info[0:2])
@property
def python_name(self):
version_info = self.version_info
return "python{}.{}".format(version_info.major, version_info.minor)
@property
def is_old_virtualenv(self):
return self.real_prefix is not None
@property
def is_venv(self):
return self.base_prefix is not None and self.version_info.major == 3
def sysconfig_path(self, key, config_var=None, sep=os.sep):
pattern = self.sysconfig_paths[key]
if config_var is None:
config_var = self.sysconfig_vars
else:
base = {k: v for k, v in self.sysconfig_vars.items()}
base.update(config_var)
config_var = base
return pattern.format(**config_var).replace("/", sep)
def creators(self, refresh=False):
if self._creators is None or refresh is True:
from virtualenv.run.plugin.creators import CreatorSelector
self._creators = CreatorSelector.for_interpreter(self)
return self._creators
@property
def system_include(self):
path = self.sysconfig_path(
"include",
{k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()},
)
if not os.path.exists(path): # some broken packaging don't respect the sysconfig, fallback to distutils path
# the pattern include the distribution name too at the end, remove that via the parent call
fallback = os.path.join(self.prefix, os.path.dirname(self.install_path("headers")))
if os.path.exists(fallback):
path = fallback
return path
@property
def system_prefix(self):
return self.real_prefix or self.base_prefix or self.prefix
@property
def system_exec_prefix(self):
return self.real_prefix or self.base_exec_prefix or self.exec_prefix
def __unicode__(self):
content = repr(self)
if sys.version_info == 2:
content = content.decode("utf-8")
return content
def __repr__(self):
return "{}({!r})".format(
self.__class__.__name__,
{k: v for k, v in self.__dict__.items() if not k.startswith("_")},
)
def __str__(self):
content = "{}({})".format(
self.__class__.__name__,
", ".join(
"{}={}".format(k, v)
for k, v in (
("spec", self.spec),
(
"system"
if self.system_executable is not None and self.system_executable != self.executable
else None,
self.system_executable,
),
(
"original"
if (
self.original_executable != self.system_executable
and self.original_executable != self.executable
)
else None,
self.original_executable,
),
("exe", self.executable),
("platform", self.platform),
("version", repr(self.version)),
("encoding_fs_io", "{}-{}".format(self.file_system_encoding, self.stdout_encoding)),
)
if k is not None
),
)
return content
@property
def spec(self):
return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture)
@classmethod
def clear_cache(cls, app_data):
# this method is not used by itself, so here and called functions can import stuff locally
from virtualenv.discovery.cached_py_info import clear
clear(app_data)
cls._cache_exe_discovery.clear()
def satisfies(self, spec, impl_must_match):
"""check if a given specification can be satisfied by the this python interpreter instance"""
if spec.path:
if self.executable == os.path.abspath(spec.path):
return True # if the path is a our own executable path we're done
if not spec.is_abs:
# if path set, and is not our original executable name, this does not match
basename = os.path.basename(self.original_executable)
spec_path = spec.path
if sys.platform == "win32":
basename, suffix = os.path.splitext(basename)
if spec_path.endswith(suffix):
spec_path = spec_path[: -len(suffix)]
if basename != spec_path:
return False
if impl_must_match:
if spec.implementation is not None and spec.implementation.lower() != self.implementation.lower():
return False
if spec.architecture is not None and spec.architecture != self.architecture:
return False
for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)):
if req is not None and our is not None and our != req:
return False
return True
_current_system = None
_current = None
@classmethod
def current(cls, app_data=None):
"""
This locates the current host interpreter information. This might be different than what we run into in case
the host python has been upgraded from underneath us.
"""
if cls._current is None:
cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False)
return cls._current
@classmethod
def current_system(cls, app_data=None):
"""
This locates the current host interpreter information. This might be different than what we run into in case
the host python has been upgraded from underneath us.
"""
if cls._current_system is None:
cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True)
return cls._current_system
def _to_json(self):
# don't save calculated paths, as these are non primitive types
return json.dumps(self._to_dict(), indent=2)
def _to_dict(self):
data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)}
# noinspection PyProtectedMember
data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary
return data
@classmethod
def from_exe(cls, exe, app_data=None, raise_on_error=True, ignore_cache=False, resolve_to_host=True, env=None):
"""Given a path to an executable get the python information"""
# this method is not used by itself, so here and called functions can import stuff locally
from virtualenv.discovery.cached_py_info import from_exe
env = os.environ if env is None else env
proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache)
# noinspection PyProtectedMember
if isinstance(proposed, PythonInfo) and resolve_to_host:
try:
proposed = proposed._resolve_to_system(app_data, proposed)
except Exception as exception:
if raise_on_error:
raise exception
logging.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception)
proposed = None
return proposed
@classmethod
def _from_json(cls, payload):
# the dictionary unroll here is to protect against pypy bug of interpreter crashing
raw = json.loads(payload)
return cls._from_dict({k: v for k, v in raw.items()})
@classmethod
def _from_dict(cls, data):
data["version_info"] = VersionInfo(**data["version_info"]) # restore this to a named tuple structure
result = cls()
result.__dict__ = {k: v for k, v in data.items()}
return result
@classmethod
def _resolve_to_system(cls, app_data, target):
start_executable = target.executable
prefixes = OrderedDict()
while target.system_executable is None:
prefix = target.real_prefix or target.base_prefix or target.prefix
if prefix in prefixes:
if len(prefixes) == 1:
# if we're linking back to ourselves accept ourselves with a WARNING
logging.info("%r links back to itself via prefixes", target)
target.system_executable = target.executable
break
for at, (p, t) in enumerate(prefixes.items(), start=1):
logging.error("%d: prefix=%s, info=%r", at, p, t)
logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target)
raise RuntimeError("prefixes are causing a circle {}".format("|".join(prefixes.keys())))
prefixes[prefix] = target
target = target.discover_exe(app_data, prefix=prefix, exact=False)
if target.executable != target.system_executable:
target = cls.from_exe(target.system_executable, app_data)
target.executable = start_executable
return target
_cache_exe_discovery = {}
def discover_exe(self, app_data, prefix, exact=True, env=None):
key = prefix, exact
if key in self._cache_exe_discovery and prefix:
logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key])
return self._cache_exe_discovery[key]
logging.debug("discover exe for %s in %s", self, prefix)
# we don't know explicitly here, do some guess work - our executable name should tell
possible_names = self._find_possible_exe_names()
possible_folders = self._find_possible_folders(prefix)
discovered = []
env = os.environ if env is None else env
for folder in possible_folders:
for name in possible_names:
info = self._check_exe(app_data, folder, name, exact, discovered, env)
if info is not None:
self._cache_exe_discovery[key] = info
return info
if exact is False and discovered:
info = self._select_most_likely(discovered, self)
folders = os.pathsep.join(possible_folders)
self._cache_exe_discovery[key] = info
logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders)
return info
msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders))
raise RuntimeError(msg)
def _check_exe(self, app_data, folder, name, exact, discovered, env):
exe_path = os.path.join(folder, name)
if not os.path.exists(exe_path):
return None
info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False, env=env)
if info is None: # ignore if for some reason we can't query
return None
for item in ["implementation", "architecture", "version_info"]:
found = getattr(info, item)
searched = getattr(self, item)
if found != searched:
if item == "version_info":
found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched)
executable = info.executable
logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched)
if exact is False:
discovered.append(info)
break
else:
return info
return None
@staticmethod
def _select_most_likely(discovered, target):
# no exact match found, start relaxing our requirements then to facilitate system package upgrades that
# could cause this (when using copy strategy of the host python)
def sort_by(info):
# we need to setup some priority of traits, this is as follows:
# implementation, major, minor, micro, architecture, tag, serial
matches = [
info.implementation == target.implementation,
info.version_info.major == target.version_info.major,
info.version_info.minor == target.version_info.minor,
info.architecture == target.architecture,
info.version_info.micro == target.version_info.micro,
info.version_info.releaselevel == target.version_info.releaselevel,
info.version_info.serial == target.version_info.serial,
]
priority = sum((1 << pos if match else 0) for pos, match in enumerate(reversed(matches)))
return priority
sorted_discovered = sorted(discovered, key=sort_by, reverse=True) # sort by priority in decreasing order
most_likely = sorted_discovered[0]
return most_likely
def _find_possible_folders(self, inside_folder):
candidate_folder = OrderedDict()
executables = OrderedDict()
executables[os.path.realpath(self.executable)] = None
executables[self.executable] = None
executables[os.path.realpath(self.original_executable)] = None
executables[self.original_executable] = None
for exe in executables.keys():
base = os.path.dirname(exe)
# following path pattern of the current
if base.startswith(self.prefix):
relative = base[len(self.prefix) :]
candidate_folder["{}{}".format(inside_folder, relative)] = None
# or at root level
candidate_folder[inside_folder] = None
return list(i for i in candidate_folder.keys() if os.path.exists(i))
def _find_possible_exe_names(self):
name_candidate = OrderedDict()
for name in self._possible_base():
for at in (3, 2, 1, 0):
version = ".".join(str(i) for i in self.version_info[:at])
for arch in ["-{}".format(self.architecture), ""]:
for ext in EXTENSIONS:
candidate = "{}{}{}{}".format(name, version, arch, ext)
name_candidate[candidate] = None
return list(name_candidate.keys())
def _possible_base(self):
possible_base = OrderedDict()
basename = os.path.splitext(os.path.basename(self.executable))[0].rstrip(digits)
possible_base[basename] = None
possible_base[self.implementation] = None
# python is always the final option as in practice is used by multiple implementation as exe name
if "python" in possible_base:
del possible_base["python"]
possible_base["python"] = None
for base in possible_base:
lower = base.lower()
yield lower
from virtualenv.info import fs_is_case_sensitive
if fs_is_case_sensitive():
if base != lower:
yield base
upper = base.upper()
if upper != base:
yield upper
if __name__ == "__main__":
# dump a JSON representation of the current python
# noinspection PyProtectedMember
print(PythonInfo()._to_json())

View file

@ -0,0 +1,122 @@
"""A Python specification is an abstract requirement definition of a interpreter"""
from __future__ import absolute_import, unicode_literals
import os
import re
import sys
from collections import OrderedDict
from virtualenv.info import fs_is_case_sensitive
from virtualenv.util.six import ensure_str
PATTERN = re.compile(r"^(?P<impl>[a-zA-Z]+)?(?P<version>[0-9.]+)?(?:-(?P<arch>32|64))?$")
IS_WIN = sys.platform == "win32"
class PythonSpec(object):
"""Contains specification about a Python Interpreter"""
def __init__(self, str_spec, implementation, major, minor, micro, architecture, path):
self.str_spec = str_spec
self.implementation = implementation
self.major = major
self.minor = minor
self.micro = micro
self.architecture = architecture
self.path = path
@classmethod
def from_string_spec(cls, string_spec):
impl, major, minor, micro, arch, path = None, None, None, None, None, None
if os.path.isabs(string_spec):
path = string_spec
else:
ok = False
match = re.match(PATTERN, string_spec)
if match:
def _int_or_none(val):
return None if val is None else int(val)
try:
groups = match.groupdict()
version = groups["version"]
if version is not None:
versions = tuple(int(i) for i in version.split(".") if i)
if len(versions) > 3:
raise ValueError
if len(versions) == 3:
major, minor, micro = versions
elif len(versions) == 2:
major, minor = versions
elif len(versions) == 1:
version_data = versions[0]
major = int(str(version_data)[0]) # first digit major
if version_data > 9:
minor = int(str(version_data)[1:])
ok = True
except ValueError:
pass
else:
impl = groups["impl"]
if impl == "py" or impl == "python":
impl = "CPython"
arch = _int_or_none(groups["arch"])
if not ok:
path = string_spec
return cls(string_spec, impl, major, minor, micro, arch, path)
def generate_names(self):
impls = OrderedDict()
if self.implementation:
# first consider implementation as it is
impls[self.implementation] = False
if fs_is_case_sensitive():
# for case sensitive file systems consider lower and upper case versions too
# trivia: MacBooks and all pre 2018 Windows-es were case insensitive by default
impls[self.implementation.lower()] = False
impls[self.implementation.upper()] = False
impls["python"] = True # finally consider python as alias, implementation must match now
version = self.major, self.minor, self.micro
try:
version = version[: version.index(None)]
except ValueError:
pass
for impl, match in impls.items():
for at in range(len(version), -1, -1):
cur_ver = version[0:at]
spec = "{}{}".format(impl, ".".join(str(i) for i in cur_ver))
yield spec, match
@property
def is_abs(self):
return self.path is not None and os.path.isabs(self.path)
def satisfies(self, spec):
"""called when there's a candidate metadata spec to see if compatible - e.g. PEP-514 on Windows"""
if spec.is_abs and self.is_abs and self.path != spec.path:
return False
if spec.implementation is not None and spec.implementation.lower() != self.implementation.lower():
return False
if spec.architecture is not None and spec.architecture != self.architecture:
return False
for our, req in zip((self.major, self.minor, self.micro), (spec.major, spec.minor, spec.micro)):
if req is not None and our is not None and our != req:
return False
return True
def __unicode__(self):
return "{}({})".format(
type(self).__name__,
", ".join(
"{}={}".format(k, getattr(self, k))
for k in ("implementation", "major", "minor", "micro", "architecture", "path")
if getattr(self, k) is not None
),
)
def __repr__(self):
return ensure_str(self.__unicode__())

View file

@ -0,0 +1,31 @@
from __future__ import absolute_import, unicode_literals
from ..py_info import PythonInfo
from ..py_spec import PythonSpec
from .pep514 import discover_pythons
class Pep514PythonInfo(PythonInfo):
""" """
def propose_interpreters(spec, cache_dir, env):
# see if PEP-514 entries are good
# start with higher python versions in an effort to use the latest version available
# and prefer PythonCore over conda pythons (as virtualenv is mostly used by non conda tools)
existing = list(discover_pythons())
existing.sort(
key=lambda i: tuple(-1 if j is None else j for j in i[1:4]) + (1 if i[0] == "PythonCore" else 0,), reverse=True
)
for name, major, minor, arch, exe, _ in existing:
# pre-filter
if name in ("PythonCore", "ContinuumAnalytics"):
name = "CPython"
registry_spec = PythonSpec(None, name, major, minor, None, arch, exe)
if registry_spec.satisfies(spec):
interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False)
if interpreter is not None:
if interpreter.satisfies(spec, impl_must_match=True):
yield interpreter

View file

@ -0,0 +1,161 @@
"""Implement https://www.python.org/dev/peps/pep-0514/ to discover interpreters - Windows only"""
from __future__ import absolute_import, print_function, unicode_literals
import os
import re
from logging import basicConfig, getLogger
import six
if six.PY3:
import winreg
else:
# noinspection PyUnresolvedReferences
import _winreg as winreg
LOGGER = getLogger(__name__)
def enum_keys(key):
at = 0
while True:
try:
yield winreg.EnumKey(key, at)
except OSError:
break
at += 1
def get_value(key, value_name):
try:
return winreg.QueryValueEx(key, value_name)[0]
except OSError:
return None
def discover_pythons():
for hive, hive_name, key, flags, default_arch in [
(winreg.HKEY_CURRENT_USER, "HKEY_CURRENT_USER", r"Software\Python", 0, 64),
(winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE", r"Software\Python", winreg.KEY_WOW64_64KEY, 64),
(winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE", r"Software\Python", winreg.KEY_WOW64_32KEY, 32),
]:
for spec in process_set(hive, hive_name, key, flags, default_arch):
yield spec
def process_set(hive, hive_name, key, flags, default_arch):
try:
with winreg.OpenKeyEx(hive, key, 0, winreg.KEY_READ | flags) as root_key:
for company in enum_keys(root_key):
if company == "PyLauncher": # reserved
continue
for spec in process_company(hive_name, company, root_key, default_arch):
yield spec
except OSError:
pass
def process_company(hive_name, company, root_key, default_arch):
with winreg.OpenKeyEx(root_key, company) as company_key:
for tag in enum_keys(company_key):
spec = process_tag(hive_name, company, company_key, tag, default_arch)
if spec is not None:
yield spec
def process_tag(hive_name, company, company_key, tag, default_arch):
with winreg.OpenKeyEx(company_key, tag) as tag_key:
version = load_version_data(hive_name, company, tag, tag_key)
if version is not None: # if failed to get version bail
major, minor, _ = version
arch = load_arch_data(hive_name, company, tag, tag_key, default_arch)
if arch is not None:
exe_data = load_exe(hive_name, company, company_key, tag)
if exe_data is not None:
exe, args = exe_data
return company, major, minor, arch, exe, args
def load_exe(hive_name, company, company_key, tag):
key_path = "{}/{}/{}".format(hive_name, company, tag)
try:
with winreg.OpenKeyEx(company_key, r"{}\InstallPath".format(tag)) as ip_key:
with ip_key:
exe = get_value(ip_key, "ExecutablePath")
if exe is None:
ip = get_value(ip_key, None)
if ip is None:
msg(key_path, "no ExecutablePath or default for it")
else:
exe = os.path.join(ip, str("python.exe"))
if exe is not None and os.path.exists(exe):
args = get_value(ip_key, "ExecutableArguments")
return exe, args
else:
msg(key_path, "could not load exe with value {}".format(exe))
except OSError:
msg("{}/{}".format(key_path, "InstallPath"), "missing")
return None
def load_arch_data(hive_name, company, tag, tag_key, default_arch):
arch_str = get_value(tag_key, "SysArchitecture")
if arch_str is not None:
key_path = "{}/{}/{}/SysArchitecture".format(hive_name, company, tag)
try:
return parse_arch(arch_str)
except ValueError as sys_arch:
msg(key_path, sys_arch)
return default_arch
def parse_arch(arch_str):
if isinstance(arch_str, six.string_types):
match = re.match(r"^(\d+)bit$", arch_str)
if match:
return int(next(iter(match.groups())))
error = "invalid format {}".format(arch_str)
else:
error = "arch is not string: {}".format(repr(arch_str))
raise ValueError(error)
def load_version_data(hive_name, company, tag, tag_key):
for candidate, key_path in [
(get_value(tag_key, "SysVersion"), "{}/{}/{}/SysVersion".format(hive_name, company, tag)),
(tag, "{}/{}/{}".format(hive_name, company, tag)),
]:
if candidate is not None:
try:
return parse_version(candidate)
except ValueError as sys_version:
msg(key_path, sys_version)
return None
def parse_version(version_str):
if isinstance(version_str, six.string_types):
match = re.match(r"^(\d+)(?:\.(\d+))?(?:\.(\d+))?$", version_str)
if match:
return tuple(int(i) if i is not None else None for i in match.groups())
error = "invalid format {}".format(version_str)
else:
error = "version is not string: {}".format(repr(version_str))
raise ValueError(error)
def msg(path, what):
LOGGER.warning("PEP-514 violation in Windows Registry at {} error: {}".format(path, what))
def _run():
basicConfig()
interpreters = []
for spec in discover_pythons():
interpreters.append(repr(spec))
print("\n".join(sorted(interpreters)))
if __name__ == "__main__":
_run()