This commit is contained in:
Waylon Walker 2022-03-31 20:20:07 -05:00
commit 38355d2442
No known key found for this signature in database
GPG key ID: 66E2BF2B4190EFE4
9083 changed files with 1225834 additions and 0 deletions

View file

@ -0,0 +1,11 @@
from __future__ import absolute_import, unicode_literals
import sys
if sys.version_info[0] == 3:
import configparser as ConfigParser
else:
import ConfigParser
__all__ = ("ConfigParser",)

View file

@ -0,0 +1,13 @@
"""Errors"""
from __future__ import absolute_import, unicode_literals
class ProcessCallFailed(RuntimeError):
"""Failed a process call"""
def __init__(self, code, out, err, cmd):
super(ProcessCallFailed, self).__init__(code, out, err, cmd)
self.code = code
self.out = out
self.err = err
self.cmd = cmd

View file

@ -0,0 +1,171 @@
"""holds locking functionality that works across processes"""
from __future__ import absolute_import, unicode_literals
import logging
import os
from abc import ABCMeta, abstractmethod
from contextlib import contextmanager
from threading import Lock, RLock
from filelock import FileLock, Timeout
from six import add_metaclass
from virtualenv.util.path import Path
class _CountedFileLock(FileLock):
def __init__(self, lock_file):
parent = os.path.dirname(lock_file)
if not os.path.isdir(parent):
try:
os.makedirs(parent)
except OSError:
pass
super(_CountedFileLock, self).__init__(lock_file)
self.count = 0
self.thread_safe = RLock()
def acquire(self, timeout=None, poll_interval=0.05):
with self.thread_safe:
if self.count == 0:
super(_CountedFileLock, self).acquire(timeout, poll_interval)
self.count += 1
def release(self, force=False):
with self.thread_safe:
if self.count == 1:
super(_CountedFileLock, self).release(force=force)
self.count = max(self.count - 1, 0)
_lock_store = {}
_store_lock = Lock()
@add_metaclass(ABCMeta)
class PathLockBase(object):
def __init__(self, folder):
path = Path(folder)
self.path = path.resolve() if path.exists() else path
def __repr__(self):
return "{}({})".format(self.__class__.__name__, self.path)
def __div__(self, other):
return type(self)(self.path / other)
def __truediv__(self, other):
return self.__div__(other)
@abstractmethod
def __enter__(self):
raise NotImplementedError
@abstractmethod
def __exit__(self, exc_type, exc_val, exc_tb):
raise NotImplementedError
@abstractmethod
@contextmanager
def lock_for_key(self, name, no_block=False):
raise NotImplementedError
@abstractmethod
@contextmanager
def non_reentrant_lock_for_key(name):
raise NotImplementedError
class ReentrantFileLock(PathLockBase):
def __init__(self, folder):
super(ReentrantFileLock, self).__init__(folder)
self._lock = None
def _create_lock(self, name=""):
lock_file = str(self.path / "{}.lock".format(name))
with _store_lock:
if lock_file not in _lock_store:
_lock_store[lock_file] = _CountedFileLock(lock_file)
return _lock_store[lock_file]
@staticmethod
def _del_lock(lock):
if lock is not None:
with _store_lock:
with lock.thread_safe:
if lock.count == 0:
_lock_store.pop(lock.lock_file, None)
def __del__(self):
self._del_lock(self._lock)
def __enter__(self):
self._lock = self._create_lock()
self._lock_file(self._lock)
def __exit__(self, exc_type, exc_val, exc_tb):
self._release(self._lock)
self._del_lock(self._lock)
self._lock = None
def _lock_file(self, lock, no_block=False):
# multiple processes might be trying to get a first lock... so we cannot check if this directory exist without
# a lock, but that lock might then become expensive, and it's not clear where that lock should live.
# Instead here we just ignore if we fail to create the directory.
try:
os.makedirs(str(self.path))
except OSError:
pass
try:
lock.acquire(0.0001)
except Timeout:
if no_block:
raise
logging.debug("lock file %s present, will block until released", lock.lock_file)
lock.release() # release the acquire try from above
lock.acquire()
@staticmethod
def _release(lock):
lock.release()
@contextmanager
def lock_for_key(self, name, no_block=False):
lock = self._create_lock(name)
try:
try:
self._lock_file(lock, no_block)
yield
finally:
self._release(lock)
finally:
self._del_lock(lock)
lock = None
@contextmanager
def non_reentrant_lock_for_key(self, name):
with _CountedFileLock(str(self.path / "{}.lock".format(name))):
yield
class NoOpFileLock(PathLockBase):
def __enter__(self):
raise NotImplementedError
def __exit__(self, exc_type, exc_val, exc_tb):
raise NotImplementedError
@contextmanager
def lock_for_key(self, name, no_block=False):
yield
@contextmanager
def non_reentrant_lock_for_key(self, name):
yield
__all__ = (
"NoOpFileLock",
"ReentrantFileLock",
"Timeout",
)

View file

@ -0,0 +1,18 @@
from __future__ import absolute_import, unicode_literals
from ._pathlib import Path
from ._permission import make_exe, set_tree
from ._sync import copy, copytree, ensure_dir, safe_delete, symlink
from ._win import get_short_path_name
__all__ = (
"ensure_dir",
"symlink",
"copy",
"copytree",
"Path",
"make_exe",
"set_tree",
"safe_delete",
"get_short_path_name",
)

View file

@ -0,0 +1,17 @@
from __future__ import absolute_import, unicode_literals
import sys
import six
if six.PY3:
from pathlib import Path
else:
if sys.platform == "win32":
# workaround for https://github.com/mcmtroffaes/pathlib2/issues/56
from .via_os_path import Path
else:
from pathlib2 import Path
__all__ = ("Path",)

View file

@ -0,0 +1,151 @@
from __future__ import absolute_import, unicode_literals
import os
import platform
from contextlib import contextmanager
from virtualenv.util.six import ensure_str, ensure_text
IS_PYPY = platform.python_implementation() == "PyPy"
class Path(object):
def __init__(self, path):
if isinstance(path, Path):
_path = path._path
else:
_path = ensure_text(path)
if IS_PYPY:
_path = _path.encode("utf-8")
self._path = _path
def __repr__(self):
return ensure_str("Path({})".format(ensure_text(self._path)))
def __unicode__(self):
return ensure_text(self._path)
def __str__(self):
return ensure_str(self._path)
def __div__(self, other):
if isinstance(other, Path):
right = other._path
else:
right = ensure_text(other)
if IS_PYPY:
right = right.encode("utf-8")
return Path(os.path.join(self._path, right))
def __truediv__(self, other):
return self.__div__(other)
def __eq__(self, other):
return self._path == (other._path if isinstance(other, Path) else None)
def __ne__(self, other):
return not (self == other)
def __hash__(self):
return hash(self._path)
def as_posix(self):
return str(self).replace(os.sep, "/")
def exists(self):
return os.path.exists(self._path)
@property
def parent(self):
return Path(os.path.abspath(os.path.join(self._path, os.path.pardir)))
def resolve(self):
return Path(os.path.realpath(self._path))
@property
def name(self):
return os.path.basename(self._path)
@property
def parts(self):
return self._path.split(os.sep)
def is_file(self):
return os.path.isfile(self._path)
def is_dir(self):
return os.path.isdir(self._path)
def mkdir(self, parents=True, exist_ok=True):
try:
os.makedirs(self._path)
except OSError:
if not exist_ok:
raise
def read_text(self, encoding="utf-8"):
return self.read_bytes().decode(encoding)
def read_bytes(self):
with open(self._path, "rb") as file_handler:
return file_handler.read()
def write_bytes(self, content):
with open(self._path, "wb") as file_handler:
file_handler.write(content)
def write_text(self, text, encoding="utf-8"):
self.write_bytes(text.encode(encoding))
def iterdir(self):
for p in os.listdir(self._path):
yield Path(os.path.join(self._path, p))
@property
def suffix(self):
_, ext = os.path.splitext(self.name)
return ext
@property
def stem(self):
base, _ = os.path.splitext(self.name)
return base
@contextmanager
def open(self, mode="r"):
with open(self._path, mode) as file_handler:
yield file_handler
@property
def parents(self):
result = []
parts = self.parts
for i in range(len(parts) - 1):
result.append(Path(os.sep.join(parts[0 : i + 1])))
return result[::-1]
def unlink(self):
os.remove(self._path)
def with_name(self, name):
return self.parent / name
def is_symlink(self):
return os.path.islink(self._path)
def relative_to(self, other):
if not self._path.startswith(other._path):
raise ValueError("{} does not start with {}".format(self._path, other._path))
return Path(os.sep.join(self.parts[len(other.parts) :]))
def stat(self):
return os.stat(self._path)
def chmod(self, mode):
os.chmod(self._path, mode)
def absolute(self):
return Path(os.path.abspath(self._path))
__all__ = ("Path",)

View file

@ -0,0 +1,32 @@
from __future__ import absolute_import, unicode_literals
import os
from stat import S_IXGRP, S_IXOTH, S_IXUSR
from virtualenv.util.six import ensure_text
def make_exe(filename):
original_mode = filename.stat().st_mode
levels = [S_IXUSR, S_IXGRP, S_IXOTH]
for at in range(len(levels), 0, -1):
try:
mode = original_mode
for level in levels[:at]:
mode |= level
filename.chmod(mode)
break
except OSError:
continue
def set_tree(folder, stat):
for root, _, files in os.walk(ensure_text(str(folder))):
for filename in files:
os.chmod(os.path.join(root, filename), stat)
__all__ = (
"make_exe",
"set_tree",
)

View file

@ -0,0 +1,97 @@
from __future__ import absolute_import, unicode_literals
import logging
import os
import shutil
from stat import S_IWUSR
from six import PY2
from virtualenv.info import IS_CPYTHON, IS_WIN
from virtualenv.util.six import ensure_text
if PY2 and IS_CPYTHON and IS_WIN: # CPython2 on Windows supports unicode paths if passed as unicode
def norm(src):
return ensure_text(str(src))
else:
norm = str
def ensure_dir(path):
if not path.exists():
logging.debug("create folder %s", ensure_text(str(path)))
os.makedirs(norm(path))
def ensure_safe_to_do(src, dest):
if src == dest:
raise ValueError("source and destination is the same {}".format(src))
if not dest.exists():
return
if dest.is_dir() and not dest.is_symlink():
logging.debug("remove directory %s", dest)
safe_delete(dest)
else:
logging.debug("remove file %s", dest)
dest.unlink()
def symlink(src, dest):
ensure_safe_to_do(src, dest)
logging.debug("symlink %s", _Debug(src, dest))
dest.symlink_to(src, target_is_directory=src.is_dir())
def copy(src, dest):
ensure_safe_to_do(src, dest)
is_dir = src.is_dir()
method = copytree if is_dir else shutil.copy
logging.debug("copy %s", _Debug(src, dest))
method(norm(src), norm(dest))
def copytree(src, dest):
for root, _, files in os.walk(src):
dest_dir = os.path.join(dest, os.path.relpath(root, src))
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
for name in files:
src_f = os.path.join(root, name)
dest_f = os.path.join(dest_dir, name)
shutil.copy(src_f, dest_f)
def safe_delete(dest):
def onerror(func, path, exc_info):
if not os.access(path, os.W_OK):
os.chmod(path, S_IWUSR)
func(path)
else:
raise
shutil.rmtree(ensure_text(str(dest)), ignore_errors=True, onerror=onerror)
class _Debug(object):
def __init__(self, src, dest):
self.src = src
self.dest = dest
def __str__(self):
return "{}{} to {}".format(
"directory " if self.src.is_dir() else "",
ensure_text(str(self.src)),
ensure_text(str(self.dest)),
)
__all__ = (
"ensure_dir",
"symlink",
"copy",
"symlink",
"copytree",
"safe_delete",
)

View file

@ -0,0 +1,19 @@
def get_short_path_name(long_name):
"""
Gets the short path name of a given long path.
http://stackoverflow.com/a/23598461/200291
"""
import ctypes
from ctypes import wintypes
_GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
_GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD]
_GetShortPathNameW.restype = wintypes.DWORD
output_buf_size = 0
while True:
output_buf = ctypes.create_unicode_buffer(output_buf_size)
needed = _GetShortPathNameW(long_name, output_buf, output_buf_size)
if output_buf_size >= needed:
return output_buf.value
else:
output_buf_size = needed

View file

@ -0,0 +1,50 @@
"""Backward compatibility layer with older version of six.
This is used to avoid virtualenv requiring a version of six newer than what
the system may have.
"""
from __future__ import absolute_import
from six import PY2, PY3, binary_type, text_type
try:
from six import ensure_text
except ImportError:
def ensure_text(s, encoding="utf-8", errors="strict"):
"""Coerce *s* to six.text_type.
For Python 2:
- `unicode` -> `unicode`
- `str` -> `unicode`
For Python 3:
- `str` -> `str`
- `bytes` -> decoded to `str`
"""
if isinstance(s, binary_type):
return s.decode(encoding, errors)
elif isinstance(s, text_type):
return s
else:
raise TypeError("not expecting type '%s'" % type(s))
try:
from six import ensure_str
except ImportError:
def ensure_str(s, encoding="utf-8", errors="strict"):
"""Coerce *s* to `str`.
For Python 2:
- `unicode` -> encoded to `str`
- `str` -> `str`
For Python 3:
- `str` -> `str`
- `bytes` -> decoded to `str`
"""
if not isinstance(s, (text_type, binary_type)):
raise TypeError("not expecting type '%s'" % type(s))
if PY2 and isinstance(s, text_type):
s = s.encode(encoding, errors)
elif PY3 and isinstance(s, binary_type):
s = s.decode(encoding, errors)
return s

View file

@ -0,0 +1,40 @@
from __future__ import absolute_import, unicode_literals
import subprocess
import sys
import six
if six.PY2 and sys.platform == "win32":
from . import _win_subprocess
Popen = _win_subprocess.Popen
else:
Popen = subprocess.Popen
CREATE_NO_WINDOW = 0x80000000
def run_cmd(cmd):
try:
process = Popen(
cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
out, err = process.communicate() # input disabled
code = process.returncode
except OSError as os_error:
code, out, err = os_error.errno, "", os_error.strerror
return code, out, err
__all__ = (
"subprocess",
"Popen",
"run_cmd",
"CREATE_NO_WINDOW",
)

View file

@ -0,0 +1,176 @@
# flake8: noqa
# fmt: off
## issue: https://bugs.python.org/issue19264
import ctypes
import os
import platform
import subprocess
from ctypes import Structure, WinError, byref, c_char_p, c_void_p, c_wchar, c_wchar_p, sizeof, windll
from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPCWSTR, LPVOID, LPWSTR, WORD
import _subprocess
##
## Types
##
CREATE_UNICODE_ENVIRONMENT = 0x00000400
LPCTSTR = c_char_p
LPTSTR = c_wchar_p
LPSECURITY_ATTRIBUTES = c_void_p
LPBYTE = ctypes.POINTER(BYTE)
class STARTUPINFOW(Structure):
_fields_ = [
("cb", DWORD), ("lpReserved", LPWSTR),
("lpDesktop", LPWSTR), ("lpTitle", LPWSTR),
("dwX", DWORD), ("dwY", DWORD),
("dwXSize", DWORD), ("dwYSize", DWORD),
("dwXCountChars", DWORD), ("dwYCountChars", DWORD),
("dwFillAtrribute", DWORD), ("dwFlags", DWORD),
("wShowWindow", WORD), ("cbReserved2", WORD),
("lpReserved2", LPBYTE), ("hStdInput", HANDLE),
("hStdOutput", HANDLE), ("hStdError", HANDLE),
]
LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW)
class PROCESS_INFORMATION(Structure):
_fields_ = [
("hProcess", HANDLE), ("hThread", HANDLE),
("dwProcessId", DWORD), ("dwThreadId", DWORD),
]
LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION)
class DUMMY_HANDLE(ctypes.c_void_p):
def __init__(self, *a, **kw):
super(DUMMY_HANDLE, self).__init__(*a, **kw)
self.closed = False
def Close(self):
if not self.closed:
windll.kernel32.CloseHandle(self)
self.closed = True
def __int__(self):
return self.value
CreateProcessW = windll.kernel32.CreateProcessW
CreateProcessW.argtypes = [
LPCWSTR, LPWSTR, LPSECURITY_ATTRIBUTES,
LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR,
LPSTARTUPINFOW, LPPROCESS_INFORMATION,
]
CreateProcessW.restype = BOOL
##
## Patched functions/classes
##
def CreateProcess(
executable, args, _p_attr, _t_attr,
inherit_handles, creation_flags, env, cwd,
startup_info,
):
"""Create a process supporting unicode executable and args for win32
Python implementation of CreateProcess using CreateProcessW for Win32
"""
si = STARTUPINFOW(
dwFlags=startup_info.dwFlags,
wShowWindow=startup_info.wShowWindow,
cb=sizeof(STARTUPINFOW),
## XXXvlab: not sure of the casting here to ints.
hStdInput=startup_info.hStdInput if startup_info.hStdInput is None else int(startup_info.hStdInput),
hStdOutput=startup_info.hStdOutput if startup_info.hStdOutput is None else int(startup_info.hStdOutput),
hStdError=startup_info.hStdError if startup_info.hStdError is None else int(startup_info.hStdError),
)
wenv = None
if env is not None:
## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar
env = (
unicode("").join([
unicode("%s=%s\0") % (k, v)
for k, v in env.items()
])
) + unicode("\0")
wenv = (c_wchar * len(env))()
wenv.value = env
wcwd = None
if cwd is not None:
wcwd = unicode(cwd)
pi = PROCESS_INFORMATION()
creation_flags |= CREATE_UNICODE_ENVIRONMENT
if CreateProcessW(
executable, args, None, None,
inherit_handles, creation_flags,
wenv, wcwd, byref(si), byref(pi),
):
return (
DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread),
pi.dwProcessId, pi.dwThreadId,
)
raise WinError()
class Popen(subprocess.Popen):
"""This superseeds Popen and corrects a bug in cPython 2.7 implem"""
def _execute_child(
self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell, to_close,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
):
"""Code from part of _execute_child from Python 2.7 (9fbb65e)
There are only 2 little changes concerning the construction of
the the final string in shell mode: we preempt the creation of
the command string when shell is True, because original function
will try to encode unicode args which we want to avoid to be able to
sending it as-is to ``CreateProcess``.
"""
if startupinfo is None:
startupinfo = subprocess.STARTUPINFO()
if not isinstance(args, subprocess.types.StringTypes):
args = [i if isinstance(i, bytes) else i.encode('utf-8') for i in args]
args = subprocess.list2cmdline(args)
if platform.python_implementation() == "CPython":
args = args.decode('utf-8')
startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = _subprocess.SW_HIDE
env = os.environ if env is None else env
comspec = env.get("COMSPEC", unicode("cmd.exe"))
if (
_subprocess.GetVersion() >= 0x80000000 or
os.path.basename(comspec).lower() == "command.com"
):
w9xpopen = self._find_w9xpopen()
args = unicode('"%s" %s') % (w9xpopen, args)
creationflags |= _subprocess.CREATE_NEW_CONSOLE
super(Popen, self)._execute_child(
args, executable,
preexec_fn, close_fds, cwd, env, universal_newlines,
startupinfo, creationflags, False, to_close, p2cread,
p2cwrite, c2pread, c2pwrite, errread, errwrite,
)
_subprocess.CreateProcess = CreateProcess
# fmt: on

View file

@ -0,0 +1,33 @@
from __future__ import absolute_import, unicode_literals
import logging
import os
import zipfile
from virtualenv.info import IS_WIN, ROOT
from virtualenv.util.six import ensure_text
def read(full_path):
sub_file = _get_path_within_zip(full_path)
with zipfile.ZipFile(ROOT, "r") as zip_file:
with zip_file.open(sub_file) as file_handler:
return file_handler.read().decode("utf-8")
def extract(full_path, dest):
logging.debug("extract %s to %s", full_path, dest)
sub_file = _get_path_within_zip(full_path)
with zipfile.ZipFile(ROOT, "r") as zip_file:
info = zip_file.getinfo(sub_file)
info.filename = dest.name
zip_file.extract(info, ensure_text(str(dest.parent)))
def _get_path_within_zip(full_path):
full_path = os.path.abspath(str(full_path))
sub_file = full_path[len(ROOT) + 1 :]
if IS_WIN:
# paths are always UNIX separators, even on Windows, though __file__ still follows platform default
sub_file = sub_file.replace(os.sep, "/")
return sub_file