This commit is contained in:
Waylon Walker 2022-03-31 20:20:07 -05:00
commit 38355d2442
No known key found for this signature in database
GPG key ID: 66E2BF2B4190EFE4
9083 changed files with 1225834 additions and 0 deletions

View file

@ -0,0 +1,5 @@
from pre_commit.main import main
if __name__ == '__main__':
raise SystemExit(main())

View file

@ -0,0 +1,413 @@
import argparse
import functools
import logging
import re
import shlex
import sys
from typing import Any
from typing import Dict
from typing import Optional
from typing import Sequence
import cfgv
from identify.identify import ALL_TAGS
import pre_commit.constants as C
from pre_commit.color import add_color_option
from pre_commit.errors import FatalError
from pre_commit.languages.all import all_languages
from pre_commit.logging_handler import logging_handler
from pre_commit.util import parse_version
from pre_commit.util import yaml_load
logger = logging.getLogger('pre_commit')
check_string_regex = cfgv.check_and(cfgv.check_string, cfgv.check_regex)
def check_type_tag(tag: str) -> None:
if tag not in ALL_TAGS:
raise cfgv.ValidationError(
f'Type tag {tag!r} is not recognized. '
f'Try upgrading identify and pre-commit?',
)
def check_min_version(version: str) -> None:
if parse_version(version) > parse_version(C.VERSION):
raise cfgv.ValidationError(
f'pre-commit version {version} is required but version '
f'{C.VERSION} is installed. '
f'Perhaps run `pip install --upgrade pre-commit`.',
)
def _make_argparser(filenames_help: str) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
parser.add_argument('filenames', nargs='*', help=filenames_help)
parser.add_argument('-V', '--version', action='version', version=C.VERSION)
add_color_option(parser)
return parser
MANIFEST_HOOK_DICT = cfgv.Map(
'Hook', 'id',
cfgv.Required('id', cfgv.check_string),
cfgv.Required('name', cfgv.check_string),
cfgv.Required('entry', cfgv.check_string),
cfgv.Required('language', cfgv.check_one_of(all_languages)),
cfgv.Optional('alias', cfgv.check_string, ''),
cfgv.Optional('files', check_string_regex, ''),
cfgv.Optional('exclude', check_string_regex, '^$'),
cfgv.Optional('types', cfgv.check_array(check_type_tag), ['file']),
cfgv.Optional('types_or', cfgv.check_array(check_type_tag), []),
cfgv.Optional('exclude_types', cfgv.check_array(check_type_tag), []),
cfgv.Optional(
'additional_dependencies', cfgv.check_array(cfgv.check_string), [],
),
cfgv.Optional('args', cfgv.check_array(cfgv.check_string), []),
cfgv.Optional('always_run', cfgv.check_bool, False),
cfgv.Optional('fail_fast', cfgv.check_bool, False),
cfgv.Optional('pass_filenames', cfgv.check_bool, True),
cfgv.Optional('description', cfgv.check_string, ''),
cfgv.Optional('language_version', cfgv.check_string, C.DEFAULT),
cfgv.Optional('log_file', cfgv.check_string, ''),
cfgv.Optional('minimum_pre_commit_version', cfgv.check_string, '0'),
cfgv.Optional('require_serial', cfgv.check_bool, False),
cfgv.Optional('stages', cfgv.check_array(cfgv.check_one_of(C.STAGES)), []),
cfgv.Optional('verbose', cfgv.check_bool, False),
)
MANIFEST_SCHEMA = cfgv.Array(MANIFEST_HOOK_DICT)
class InvalidManifestError(FatalError):
pass
load_manifest = functools.partial(
cfgv.load_from_filename,
schema=MANIFEST_SCHEMA,
load_strategy=yaml_load,
exc_tp=InvalidManifestError,
)
def validate_manifest_main(argv: Optional[Sequence[str]] = None) -> int:
parser = _make_argparser('Manifest filenames.')
args = parser.parse_args(argv)
with logging_handler(args.color):
ret = 0
for filename in args.filenames:
try:
load_manifest(filename)
except InvalidManifestError as e:
print(e)
ret = 1
return ret
LOCAL = 'local'
META = 'meta'
# should inherit from cfgv.Conditional if sha support is dropped
class WarnMutableRev(cfgv.ConditionalOptional):
def check(self, dct: Dict[str, Any]) -> None:
super().check(dct)
if self.key in dct:
rev = dct[self.key]
if '.' not in rev and not re.match(r'^[a-fA-F0-9]+$', rev):
logger.warning(
f'The {self.key!r} field of repo {dct["repo"]!r} '
f'appears to be a mutable reference '
f'(moving tag / branch). Mutable references are never '
f'updated after first install and are not supported. '
f'See https://pre-commit.com/#using-the-latest-version-for-a-repository ' # noqa: E501
f'for more details. '
f'Hint: `pre-commit autoupdate` often fixes this.',
)
class OptionalSensibleRegexAtHook(cfgv.OptionalNoDefault):
def check(self, dct: Dict[str, Any]) -> None:
super().check(dct)
if '/*' in dct.get(self.key, ''):
logger.warning(
f'The {self.key!r} field in hook {dct.get("id")!r} is a '
f"regex, not a glob -- matching '/*' probably isn't what you "
f'want here',
)
for fwd_slash_re in (r'[\\/]', r'[\/]', r'[/\\]'):
if fwd_slash_re in dct.get(self.key, ''):
logger.warning(
fr'pre-commit normalizes slashes in the {self.key!r} '
fr'field in hook {dct.get("id")!r} to forward slashes, '
fr'so you can use / instead of {fwd_slash_re}',
)
class OptionalSensibleRegexAtTop(cfgv.OptionalNoDefault):
def check(self, dct: Dict[str, Any]) -> None:
super().check(dct)
if '/*' in dct.get(self.key, ''):
logger.warning(
f'The top-level {self.key!r} field is a regex, not a glob -- '
f"matching '/*' probably isn't what you want here",
)
for fwd_slash_re in (r'[\\/]', r'[\/]', r'[/\\]'):
if fwd_slash_re in dct.get(self.key, ''):
logger.warning(
fr'pre-commit normalizes the slashes in the top-level '
fr'{self.key!r} field to forward slashes, so you '
fr'can use / instead of {fwd_slash_re}',
)
class MigrateShaToRev:
key = 'rev'
@staticmethod
def _cond(key: str) -> cfgv.Conditional:
return cfgv.Conditional(
key, cfgv.check_string,
condition_key='repo',
condition_value=cfgv.NotIn(LOCAL, META),
ensure_absent=True,
)
def check(self, dct: Dict[str, Any]) -> None:
if dct.get('repo') in {LOCAL, META}:
self._cond('rev').check(dct)
self._cond('sha').check(dct)
elif 'sha' in dct and 'rev' in dct:
raise cfgv.ValidationError('Cannot specify both sha and rev')
elif 'sha' in dct:
self._cond('sha').check(dct)
else:
self._cond('rev').check(dct)
def apply_default(self, dct: Dict[str, Any]) -> None:
if 'sha' in dct:
dct['rev'] = dct.pop('sha')
remove_default = cfgv.Required.remove_default
def _entry(modname: str) -> str:
"""the hook `entry` is passed through `shlex.split()` by the command
runner, so to prevent issues with spaces and backslashes (on Windows)
it must be quoted here.
"""
return f'{shlex.quote(sys.executable)} -m pre_commit.meta_hooks.{modname}'
def warn_unknown_keys_root(
extra: Sequence[str],
orig_keys: Sequence[str],
dct: Dict[str, str],
) -> None:
logger.warning(f'Unexpected key(s) present at root: {", ".join(extra)}')
def warn_unknown_keys_repo(
extra: Sequence[str],
orig_keys: Sequence[str],
dct: Dict[str, str],
) -> None:
logger.warning(
f'Unexpected key(s) present on {dct["repo"]}: {", ".join(extra)}',
)
_meta = (
(
'check-hooks-apply', (
('name', 'Check hooks apply to the repository'),
('files', f'^{re.escape(C.CONFIG_FILE)}$'),
('entry', _entry('check_hooks_apply')),
),
),
(
'check-useless-excludes', (
('name', 'Check for useless excludes'),
('files', f'^{re.escape(C.CONFIG_FILE)}$'),
('entry', _entry('check_useless_excludes')),
),
),
(
'identity', (
('name', 'identity'),
('verbose', True),
('entry', _entry('identity')),
),
),
)
class NotAllowed(cfgv.OptionalNoDefault):
def check(self, dct: Dict[str, Any]) -> None:
if self.key in dct:
raise cfgv.ValidationError(f'{self.key!r} cannot be overridden')
META_HOOK_DICT = cfgv.Map(
'Hook', 'id',
cfgv.Required('id', cfgv.check_string),
cfgv.Required('id', cfgv.check_one_of(tuple(k for k, _ in _meta))),
# language must be system
cfgv.Optional('language', cfgv.check_one_of({'system'}), 'system'),
# entry cannot be overridden
NotAllowed('entry', cfgv.check_any),
*(
# default to the hook definition for the meta hooks
cfgv.ConditionalOptional(key, cfgv.check_any, value, 'id', hook_id)
for hook_id, values in _meta
for key, value in values
),
*(
# default to the "manifest" parsing
cfgv.OptionalNoDefault(item.key, item.check_fn)
# these will always be defaulted above
if item.key in {'name', 'language', 'entry'} else
item
for item in MANIFEST_HOOK_DICT.items
),
)
CONFIG_HOOK_DICT = cfgv.Map(
'Hook', 'id',
cfgv.Required('id', cfgv.check_string),
# All keys in manifest hook dict are valid in a config hook dict, but
# are optional.
# No defaults are provided here as the config is merged on top of the
# manifest.
*(
cfgv.OptionalNoDefault(item.key, item.check_fn)
for item in MANIFEST_HOOK_DICT.items
if item.key != 'id'
),
OptionalSensibleRegexAtHook('files', cfgv.check_string),
OptionalSensibleRegexAtHook('exclude', cfgv.check_string),
)
CONFIG_REPO_DICT = cfgv.Map(
'Repository', 'repo',
cfgv.Required('repo', cfgv.check_string),
cfgv.ConditionalRecurse(
'hooks', cfgv.Array(CONFIG_HOOK_DICT),
'repo', cfgv.NotIn(LOCAL, META),
),
cfgv.ConditionalRecurse(
'hooks', cfgv.Array(MANIFEST_HOOK_DICT),
'repo', LOCAL,
),
cfgv.ConditionalRecurse(
'hooks', cfgv.Array(META_HOOK_DICT),
'repo', META,
),
MigrateShaToRev(),
WarnMutableRev(
'rev',
cfgv.check_string,
'',
'repo',
cfgv.NotIn(LOCAL, META),
True,
),
cfgv.WarnAdditionalKeys(('repo', 'rev', 'hooks'), warn_unknown_keys_repo),
)
DEFAULT_LANGUAGE_VERSION = cfgv.Map(
'DefaultLanguageVersion', None,
cfgv.NoAdditionalKeys(all_languages),
*(cfgv.Optional(x, cfgv.check_string, C.DEFAULT) for x in all_languages),
)
CONFIG_SCHEMA = cfgv.Map(
'Config', None,
cfgv.RequiredRecurse('repos', cfgv.Array(CONFIG_REPO_DICT)),
cfgv.OptionalRecurse(
'default_language_version', DEFAULT_LANGUAGE_VERSION, {},
),
cfgv.Optional(
'default_stages',
cfgv.check_array(cfgv.check_one_of(C.STAGES)),
C.STAGES,
),
cfgv.Optional('files', check_string_regex, ''),
cfgv.Optional('exclude', check_string_regex, '^$'),
cfgv.Optional('fail_fast', cfgv.check_bool, False),
cfgv.Optional(
'minimum_pre_commit_version',
cfgv.check_and(cfgv.check_string, check_min_version),
'0',
),
cfgv.WarnAdditionalKeys(
(
'repos',
'default_language_version',
'default_stages',
'files',
'exclude',
'fail_fast',
'minimum_pre_commit_version',
'ci',
),
warn_unknown_keys_root,
),
OptionalSensibleRegexAtTop('files', cfgv.check_string),
OptionalSensibleRegexAtTop('exclude', cfgv.check_string),
# do not warn about configuration for pre-commit.ci
cfgv.OptionalNoDefault('ci', cfgv.check_type(dict)),
)
class InvalidConfigError(FatalError):
pass
def ordered_load_normalize_legacy_config(contents: str) -> Dict[str, Any]:
data = yaml_load(contents)
if isinstance(data, list):
logger.warning(
'normalizing pre-commit configuration to a top-level map. '
'support for top level list will be removed in a future version. '
'run: `pre-commit migrate-config` to automatically fix this.',
)
return {'repos': data}
else:
return data
load_config = functools.partial(
cfgv.load_from_filename,
schema=CONFIG_SCHEMA,
load_strategy=ordered_load_normalize_legacy_config,
exc_tp=InvalidConfigError,
)
def validate_config_main(argv: Optional[Sequence[str]] = None) -> int:
parser = _make_argparser('Config filenames.')
args = parser.parse_args(argv)
with logging_handler(args.color):
ret = 0
for filename in args.filenames:
try:
load_config(filename)
except InvalidConfigError as e:
print(e)
ret = 1
return ret

View file

@ -0,0 +1,107 @@
import argparse
import os
import sys
if sys.platform == 'win32': # pragma: no cover (windows)
def _enable() -> None:
from ctypes import POINTER
from ctypes import windll
from ctypes import WinError
from ctypes import WINFUNCTYPE
from ctypes.wintypes import BOOL
from ctypes.wintypes import DWORD
from ctypes.wintypes import HANDLE
STD_ERROR_HANDLE = -12
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
def bool_errcheck(result, func, args):
if not result:
raise WinError()
return args
GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(
('GetStdHandle', windll.kernel32), ((1, 'nStdHandle'),),
)
GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(
('GetConsoleMode', windll.kernel32),
((1, 'hConsoleHandle'), (2, 'lpMode')),
)
GetConsoleMode.errcheck = bool_errcheck
SetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, DWORD)(
('SetConsoleMode', windll.kernel32),
((1, 'hConsoleHandle'), (1, 'dwMode')),
)
SetConsoleMode.errcheck = bool_errcheck
# As of Windows 10, the Windows console supports (some) ANSI escape
# sequences, but it needs to be enabled using `SetConsoleMode` first.
#
# More info on the escape sequences supported:
# https://msdn.microsoft.com/en-us/library/windows/desktop/mt638032(v=vs.85).aspx
stderr = GetStdHandle(STD_ERROR_HANDLE)
flags = GetConsoleMode(stderr)
SetConsoleMode(stderr, flags | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
try:
_enable()
except OSError:
terminal_supports_color = False
else:
terminal_supports_color = True
else: # pragma: win32 no cover
terminal_supports_color = True
RED = '\033[41m'
GREEN = '\033[42m'
YELLOW = '\033[43;30m'
TURQUOISE = '\033[46;30m'
SUBTLE = '\033[2m'
NORMAL = '\033[m'
def format_color(text: str, color: str, use_color_setting: bool) -> str:
"""Format text with color.
Args:
text - Text to be formatted with color if `use_color`
color - The color start string
use_color_setting - Whether or not to color
"""
if use_color_setting:
return f'{color}{text}{NORMAL}'
else:
return text
COLOR_CHOICES = ('auto', 'always', 'never')
def use_color(setting: str) -> bool:
"""Choose whether to use color based on the command argument.
Args:
setting - Either `auto`, `always`, or `never`
"""
if setting not in COLOR_CHOICES:
raise ValueError(setting)
return (
setting == 'always' or (
setting == 'auto' and
sys.stderr.isatty() and
terminal_supports_color and
os.getenv('TERM') != 'dumb'
)
)
def add_color_option(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'--color', default=os.environ.get('PRE_COMMIT_COLOR', 'auto'),
type=use_color,
metavar='{' + ','.join(COLOR_CHOICES) + '}',
help='Whether to use color in output. Defaults to `%(default)s`.',
)

View file

@ -0,0 +1,194 @@
import os.path
import re
from typing import Any
from typing import Dict
from typing import List
from typing import NamedTuple
from typing import Optional
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit import git
from pre_commit import output
from pre_commit.clientlib import InvalidManifestError
from pre_commit.clientlib import load_config
from pre_commit.clientlib import load_manifest
from pre_commit.clientlib import LOCAL
from pre_commit.clientlib import META
from pre_commit.commands.migrate_config import migrate_config
from pre_commit.store import Store
from pre_commit.util import CalledProcessError
from pre_commit.util import cmd_output
from pre_commit.util import cmd_output_b
from pre_commit.util import tmpdir
from pre_commit.util import yaml_dump
from pre_commit.util import yaml_load
class RevInfo(NamedTuple):
repo: str
rev: str
frozen: Optional[str]
@classmethod
def from_config(cls, config: Dict[str, Any]) -> 'RevInfo':
return cls(config['repo'], config['rev'], None)
def update(self, tags_only: bool, freeze: bool) -> 'RevInfo':
git_cmd = ('git', *git.NO_FS_MONITOR)
if tags_only:
tag_cmd = (
*git_cmd, 'describe',
'FETCH_HEAD', '--tags', '--abbrev=0',
)
else:
tag_cmd = (
*git_cmd, 'describe',
'FETCH_HEAD', '--tags', '--exact',
)
with tmpdir() as tmp:
git.init_repo(tmp, self.repo)
cmd_output_b(
*git_cmd, 'fetch', 'origin', 'HEAD', '--tags',
cwd=tmp,
)
try:
rev = cmd_output(*tag_cmd, cwd=tmp)[1].strip()
except CalledProcessError:
cmd = (*git_cmd, 'rev-parse', 'FETCH_HEAD')
rev = cmd_output(*cmd, cwd=tmp)[1].strip()
frozen = None
if freeze:
exact_rev_cmd = (*git_cmd, 'rev-parse', rev)
exact = cmd_output(*exact_rev_cmd, cwd=tmp)[1].strip()
if exact != rev:
rev, frozen = exact, rev
return self._replace(rev=rev, frozen=frozen)
class RepositoryCannotBeUpdatedError(RuntimeError):
pass
def _check_hooks_still_exist_at_rev(
repo_config: Dict[str, Any],
info: RevInfo,
store: Store,
) -> None:
try:
path = store.clone(repo_config['repo'], info.rev)
manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE))
except InvalidManifestError as e:
raise RepositoryCannotBeUpdatedError(str(e))
# See if any of our hooks were deleted with the new commits
hooks = {hook['id'] for hook in repo_config['hooks']}
hooks_missing = hooks - {hook['id'] for hook in manifest}
if hooks_missing:
raise RepositoryCannotBeUpdatedError(
f'Cannot update because the update target is missing these '
f'hooks:\n{", ".join(sorted(hooks_missing))}',
)
REV_LINE_RE = re.compile(r'^(\s+)rev:(\s*)([\'"]?)([^\s#]+)(.*)(\r?\n)$')
def _original_lines(
path: str,
rev_infos: List[Optional[RevInfo]],
retry: bool = False,
) -> Tuple[List[str], List[int]]:
"""detect `rev:` lines or reformat the file"""
with open(path, newline='') as f:
original = f.read()
lines = original.splitlines(True)
idxs = [i for i, line in enumerate(lines) if REV_LINE_RE.match(line)]
if len(idxs) == len(rev_infos):
return lines, idxs
elif retry:
raise AssertionError('could not find rev lines')
else:
with open(path, 'w') as f:
f.write(yaml_dump(yaml_load(original)))
return _original_lines(path, rev_infos, retry=True)
def _write_new_config(path: str, rev_infos: List[Optional[RevInfo]]) -> None:
lines, idxs = _original_lines(path, rev_infos)
for idx, rev_info in zip(idxs, rev_infos):
if rev_info is None:
continue
match = REV_LINE_RE.match(lines[idx])
assert match is not None
new_rev_s = yaml_dump({'rev': rev_info.rev}, default_style=match[3])
new_rev = new_rev_s.split(':', 1)[1].strip()
if rev_info.frozen is not None:
comment = f' # frozen: {rev_info.frozen}'
elif match[5].strip().startswith('# frozen:'):
comment = ''
else:
comment = match[5]
lines[idx] = f'{match[1]}rev:{match[2]}{new_rev}{comment}{match[6]}'
with open(path, 'w', newline='') as f:
f.write(''.join(lines))
def autoupdate(
config_file: str,
store: Store,
tags_only: bool,
freeze: bool,
repos: Sequence[str] = (),
) -> int:
"""Auto-update the pre-commit config to the latest versions of repos."""
migrate_config(config_file, quiet=True)
retv = 0
rev_infos: List[Optional[RevInfo]] = []
changed = False
config = load_config(config_file)
for repo_config in config['repos']:
if repo_config['repo'] in {LOCAL, META}:
continue
info = RevInfo.from_config(repo_config)
if repos and info.repo not in repos:
rev_infos.append(None)
continue
output.write(f'Updating {info.repo} ... ')
new_info = info.update(tags_only=tags_only, freeze=freeze)
try:
_check_hooks_still_exist_at_rev(repo_config, new_info, store)
except RepositoryCannotBeUpdatedError as error:
output.write_line(error.args[0])
rev_infos.append(None)
retv = 1
continue
if new_info.rev != info.rev:
changed = True
if new_info.frozen:
updated_to = f'{new_info.frozen} (frozen)'
else:
updated_to = new_info.rev
msg = f'updating {info.rev} -> {updated_to}.'
output.write_line(msg)
rev_infos.append(new_info)
else:
output.write_line('already up to date.')
rev_infos.append(None)
if changed:
_write_new_config(config_file, rev_infos)
return retv

View file

@ -0,0 +1,14 @@
import os.path
from pre_commit import output
from pre_commit.store import Store
from pre_commit.util import rmtree
def clean(store: Store) -> int:
legacy_path = os.path.expanduser('~/.pre-commit')
for directory in (store.directory, legacy_path):
if os.path.exists(directory):
rmtree(directory)
output.write_line(f'Cleaned {directory}.')
return 0

View file

@ -0,0 +1,90 @@
import os.path
from typing import Any
from typing import Dict
from typing import Set
from typing import Tuple
import pre_commit.constants as C
from pre_commit import output
from pre_commit.clientlib import InvalidConfigError
from pre_commit.clientlib import InvalidManifestError
from pre_commit.clientlib import load_config
from pre_commit.clientlib import load_manifest
from pre_commit.clientlib import LOCAL
from pre_commit.clientlib import META
from pre_commit.store import Store
def _mark_used_repos(
store: Store,
all_repos: Dict[Tuple[str, str], str],
unused_repos: Set[Tuple[str, str]],
repo: Dict[str, Any],
) -> None:
if repo['repo'] == META:
return
elif repo['repo'] == LOCAL:
for hook in repo['hooks']:
deps = hook.get('additional_dependencies')
unused_repos.discard((
store.db_repo_name(repo['repo'], deps), C.LOCAL_REPO_VERSION,
))
else:
key = (repo['repo'], repo['rev'])
path = all_repos.get(key)
# can't inspect manifest if it isn't cloned
if path is None:
return
try:
manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE))
except InvalidManifestError:
return
else:
unused_repos.discard(key)
by_id = {hook['id']: hook for hook in manifest}
for hook in repo['hooks']:
if hook['id'] not in by_id:
continue
deps = hook.get(
'additional_dependencies',
by_id[hook['id']]['additional_dependencies'],
)
unused_repos.discard((
store.db_repo_name(repo['repo'], deps), repo['rev'],
))
def _gc_repos(store: Store) -> int:
configs = store.select_all_configs()
repos = store.select_all_repos()
# delete config paths which do not exist
dead_configs = [p for p in configs if not os.path.exists(p)]
live_configs = [p for p in configs if os.path.exists(p)]
all_repos = {(repo, ref): path for repo, ref, path in repos}
unused_repos = set(all_repos)
for config_path in live_configs:
try:
config = load_config(config_path)
except InvalidConfigError:
dead_configs.append(config_path)
continue
else:
for repo in config['repos']:
_mark_used_repos(store, all_repos, unused_repos, repo)
store.delete_configs(dead_configs)
for db_repo_name, ref in unused_repos:
store.delete_repo(db_repo_name, ref, all_repos[(db_repo_name, ref)])
return len(unused_repos)
def gc(store: Store) -> int:
with store.exclusive_lock():
repos_removed = _gc_repos(store)
output.write_line(f'{repos_removed} repo(s) removed.')
return 0

View file

@ -0,0 +1,237 @@
import argparse
import os.path
import subprocess
import sys
from typing import Optional
from typing import Sequence
from typing import Tuple
from pre_commit.commands.run import run
from pre_commit.envcontext import envcontext
from pre_commit.parse_shebang import normalize_cmd
from pre_commit.store import Store
Z40 = '0' * 40
def _run_legacy(
hook_type: str,
hook_dir: str,
args: Sequence[str],
) -> Tuple[int, bytes]:
if os.environ.get('PRE_COMMIT_RUNNING_LEGACY'):
raise SystemExit(
f"bug: pre-commit's script is installed in migration mode\n"
f'run `pre-commit install -f --hook-type {hook_type}` to fix '
f'this\n\n'
f'Please report this bug at '
f'https://github.com/pre-commit/pre-commit/issues',
)
if hook_type == 'pre-push':
stdin = sys.stdin.buffer.read()
else:
stdin = b''
# not running in legacy mode
legacy_hook = os.path.join(hook_dir, f'{hook_type}.legacy')
if not os.access(legacy_hook, os.X_OK):
return 0, stdin
with envcontext((('PRE_COMMIT_RUNNING_LEGACY', '1'),)):
cmd = normalize_cmd((legacy_hook, *args))
return subprocess.run(cmd, input=stdin).returncode, stdin
def _validate_config(
retv: int,
config: str,
skip_on_missing_config: bool,
) -> None:
if not os.path.isfile(config):
if skip_on_missing_config or os.getenv('PRE_COMMIT_ALLOW_NO_CONFIG'):
print(f'`{config}` config file not found. Skipping `pre-commit`.')
raise SystemExit(retv)
else:
print(
f'No {config} file was found\n'
f'- To temporarily silence this, run '
f'`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n'
f'- To permanently silence this, install pre-commit with the '
f'--allow-missing-config option\n'
f'- To uninstall pre-commit run `pre-commit uninstall`',
)
raise SystemExit(1)
def _ns(
hook_type: str,
color: bool,
*,
all_files: bool = False,
remote_branch: Optional[str] = None,
local_branch: Optional[str] = None,
from_ref: Optional[str] = None,
to_ref: Optional[str] = None,
remote_name: Optional[str] = None,
remote_url: Optional[str] = None,
commit_msg_filename: Optional[str] = None,
checkout_type: Optional[str] = None,
is_squash_merge: Optional[str] = None,
rewrite_command: Optional[str] = None,
) -> argparse.Namespace:
return argparse.Namespace(
color=color,
hook_stage=hook_type.replace('pre-', ''),
remote_branch=remote_branch,
local_branch=local_branch,
from_ref=from_ref,
to_ref=to_ref,
remote_name=remote_name,
remote_url=remote_url,
commit_msg_filename=commit_msg_filename,
all_files=all_files,
checkout_type=checkout_type,
is_squash_merge=is_squash_merge,
rewrite_command=rewrite_command,
files=(),
hook=None,
verbose=False,
show_diff_on_failure=False,
)
def _rev_exists(rev: str) -> bool:
return not subprocess.call(('git', 'rev-list', '--quiet', rev))
def _pre_push_ns(
color: bool,
args: Sequence[str],
stdin: bytes,
) -> Optional[argparse.Namespace]:
remote_name = args[0]
remote_url = args[1]
for line in stdin.decode().splitlines():
local_branch, local_sha, remote_branch, remote_sha = line.split()
if local_sha == Z40:
continue
elif remote_sha != Z40 and _rev_exists(remote_sha):
return _ns(
'pre-push', color,
from_ref=remote_sha, to_ref=local_sha,
remote_branch=remote_branch,
local_branch=local_branch,
remote_name=remote_name, remote_url=remote_url,
)
else:
# ancestors not found in remote
ancestors = subprocess.check_output((
'git', 'rev-list', local_sha, '--topo-order', '--reverse',
'--not', f'--remotes={remote_name}',
)).decode().strip()
if not ancestors:
continue
else:
first_ancestor = ancestors.splitlines()[0]
cmd = ('git', 'rev-list', '--max-parents=0', local_sha)
roots = set(subprocess.check_output(cmd).decode().splitlines())
if first_ancestor in roots:
# pushing the whole tree including root commit
return _ns(
'pre-push', color,
all_files=True,
remote_name=remote_name, remote_url=remote_url,
remote_branch=remote_branch,
local_branch=local_branch,
)
else:
rev_cmd = ('git', 'rev-parse', f'{first_ancestor}^')
source = subprocess.check_output(rev_cmd).decode().strip()
return _ns(
'pre-push', color,
from_ref=source, to_ref=local_sha,
remote_name=remote_name, remote_url=remote_url,
remote_branch=remote_branch,
local_branch=local_branch,
)
# nothing to push
return None
_EXPECTED_ARG_LENGTH_BY_HOOK = {
'commit-msg': 1,
'post-checkout': 3,
'post-commit': 0,
'pre-commit': 0,
'pre-merge-commit': 0,
'post-merge': 1,
'post-rewrite': 1,
'pre-push': 2,
}
def _check_args_length(hook_type: str, args: Sequence[str]) -> None:
if hook_type == 'prepare-commit-msg':
if len(args) < 1 or len(args) > 3:
raise SystemExit(
f'hook-impl for {hook_type} expected 1, 2, or 3 arguments '
f'but got {len(args)}: {args}',
)
elif hook_type in _EXPECTED_ARG_LENGTH_BY_HOOK:
expected = _EXPECTED_ARG_LENGTH_BY_HOOK[hook_type]
if len(args) != expected:
arguments_s = 'argument' if expected == 1 else 'arguments'
raise SystemExit(
f'hook-impl for {hook_type} expected {expected} {arguments_s} '
f'but got {len(args)}: {args}',
)
else:
raise AssertionError(f'unexpected hook type: {hook_type}')
def _run_ns(
hook_type: str,
color: bool,
args: Sequence[str],
stdin: bytes,
) -> Optional[argparse.Namespace]:
_check_args_length(hook_type, args)
if hook_type == 'pre-push':
return _pre_push_ns(color, args, stdin)
elif hook_type in {'commit-msg', 'prepare-commit-msg'}:
return _ns(hook_type, color, commit_msg_filename=args[0])
elif hook_type in {'post-commit', 'pre-merge-commit', 'pre-commit'}:
return _ns(hook_type, color)
elif hook_type == 'post-checkout':
return _ns(
hook_type, color,
from_ref=args[0], to_ref=args[1], checkout_type=args[2],
)
elif hook_type == 'post-merge':
return _ns(hook_type, color, is_squash_merge=args[0])
elif hook_type == 'post-rewrite':
return _ns(hook_type, color, rewrite_command=args[0])
else:
raise AssertionError(f'unexpected hook type: {hook_type}')
def hook_impl(
store: Store,
*,
config: str,
color: bool,
hook_type: str,
hook_dir: str,
skip_on_missing_config: bool,
args: Sequence[str],
) -> int:
retv, stdin = _run_legacy(hook_type, hook_dir, args)
_validate_config(retv, config, skip_on_missing_config)
ns = _run_ns(hook_type, color, args, stdin)
if ns is None:
return retv
else:
return retv | run(config, store, ns)

View file

@ -0,0 +1,38 @@
import logging
import os.path
from typing import Sequence
from pre_commit.commands.install_uninstall import install
from pre_commit.store import Store
from pre_commit.util import CalledProcessError
from pre_commit.util import cmd_output
logger = logging.getLogger('pre_commit')
def init_templatedir(
config_file: str,
store: Store,
directory: str,
hook_types: Sequence[str],
skip_on_missing_config: bool = True,
) -> int:
install(
config_file,
store,
hook_types=hook_types,
overwrite=True,
skip_on_missing_config=skip_on_missing_config,
git_dir=directory,
)
try:
_, out, _ = cmd_output('git', 'config', 'init.templateDir')
except CalledProcessError:
configured_path = None
else:
configured_path = os.path.realpath(os.path.expanduser(out.strip()))
dest = os.path.realpath(directory)
if configured_path != dest:
logger.warning('`init.templateDir` not set to the target directory')
logger.warning(f'maybe `git config --global init.templateDir {dest}`?')
return 0

View file

@ -0,0 +1,156 @@
import logging
import os.path
import shlex
import shutil
import sys
from typing import Optional
from typing import Sequence
from typing import Tuple
from pre_commit import git
from pre_commit import output
from pre_commit.clientlib import load_config
from pre_commit.repository import all_hooks
from pre_commit.repository import install_hook_envs
from pre_commit.store import Store
from pre_commit.util import make_executable
from pre_commit.util import resource_text
logger = logging.getLogger(__name__)
# This is used to identify the hook file we install
PRIOR_HASHES = (
b'4d9958c90bc262f47553e2c073f14cfe',
b'd8ee923c46731b42cd95cc869add4062',
b'49fd668cb42069aa1b6048464be5d395',
b'79f09a650522a87b0da915d0d983b2de',
b'e358c9dae00eac5d06b38dfdb1e33a8c',
)
CURRENT_HASH = b'138fd403232d2ddd5efb44317e38bf03'
TEMPLATE_START = '# start templated\n'
TEMPLATE_END = '# end templated\n'
def _hook_paths(
hook_type: str,
git_dir: Optional[str] = None,
) -> Tuple[str, str]:
git_dir = git_dir if git_dir is not None else git.get_git_dir()
pth = os.path.join(git_dir, 'hooks', hook_type)
return pth, f'{pth}.legacy'
def is_our_script(filename: str) -> bool:
if not os.path.exists(filename): # pragma: win32 no cover (symlink)
return False
with open(filename, 'rb') as f:
contents = f.read()
return any(h in contents for h in (CURRENT_HASH,) + PRIOR_HASHES)
def _install_hook_script(
config_file: str,
hook_type: str,
overwrite: bool = False,
skip_on_missing_config: bool = False,
git_dir: Optional[str] = None,
) -> None:
hook_path, legacy_path = _hook_paths(hook_type, git_dir=git_dir)
os.makedirs(os.path.dirname(hook_path), exist_ok=True)
# If we have an existing hook, move it to pre-commit.legacy
if os.path.lexists(hook_path) and not is_our_script(hook_path):
shutil.move(hook_path, legacy_path)
# If we specify overwrite, we simply delete the legacy file
if overwrite and os.path.exists(legacy_path):
os.remove(legacy_path)
elif os.path.exists(legacy_path):
output.write_line(
f'Running in migration mode with existing hooks at {legacy_path}\n'
f'Use -f to use only pre-commit.',
)
args = ['hook-impl', f'--config={config_file}', f'--hook-type={hook_type}']
if skip_on_missing_config:
args.append('--skip-on-missing-config')
with open(hook_path, 'w') as hook_file:
contents = resource_text('hook-tmpl')
before, rest = contents.split(TEMPLATE_START)
_, after = rest.split(TEMPLATE_END)
# on windows always use `/bin/sh` since `bash` might not be on PATH
# though we use bash-specific features `sh` on windows is actually
# bash in "POSIXLY_CORRECT" mode which still supports the features we
# use: subshells / arrays
if sys.platform == 'win32': # pragma: win32 cover
hook_file.write('#!/bin/sh\n')
hook_file.write(before + TEMPLATE_START)
hook_file.write(f'INSTALL_PYTHON={shlex.quote(sys.executable)}\n')
# TODO: python3.8+: shlex.join
args_s = ' '.join(shlex.quote(part) for part in args)
hook_file.write(f'ARGS=({args_s})\n')
hook_file.write(TEMPLATE_END + after)
make_executable(hook_path)
output.write_line(f'pre-commit installed at {hook_path}')
def install(
config_file: str,
store: Store,
hook_types: Sequence[str],
overwrite: bool = False,
hooks: bool = False,
skip_on_missing_config: bool = False,
git_dir: Optional[str] = None,
) -> int:
if git_dir is None and git.has_core_hookpaths_set():
logger.error(
'Cowardly refusing to install hooks with `core.hooksPath` set.\n'
'hint: `git config --unset-all core.hooksPath`',
)
return 1
for hook_type in hook_types:
_install_hook_script(
config_file, hook_type,
overwrite=overwrite,
skip_on_missing_config=skip_on_missing_config,
git_dir=git_dir,
)
if hooks:
install_hooks(config_file, store)
return 0
def install_hooks(config_file: str, store: Store) -> int:
install_hook_envs(all_hooks(load_config(config_file), store), store)
return 0
def _uninstall_hook_script(hook_type: str) -> None:
hook_path, legacy_path = _hook_paths(hook_type)
# If our file doesn't exist or it isn't ours, gtfo.
if not os.path.exists(hook_path) or not is_our_script(hook_path):
return
os.remove(hook_path)
output.write_line(f'{hook_type} uninstalled')
if os.path.exists(legacy_path):
os.replace(legacy_path, hook_path)
output.write_line(f'Restored previous hooks to {hook_path}')
def uninstall(hook_types: Sequence[str]) -> int:
for hook_type in hook_types:
_uninstall_hook_script(hook_type)
return 0

View file

@ -0,0 +1,55 @@
import re
import textwrap
import yaml
from pre_commit.util import yaml_load
def _is_header_line(line: str) -> bool:
return line.startswith(('#', '---')) or not line.strip()
def _migrate_map(contents: str) -> str:
if isinstance(yaml_load(contents), list):
# Find the first non-header line
lines = contents.splitlines(True)
i = 0
# Only loop on non empty configuration file
while i < len(lines) and _is_header_line(lines[i]):
i += 1
header = ''.join(lines[:i])
rest = ''.join(lines[i:])
# If they are using the "default" flow style of yaml, this operation
# will yield a valid configuration
try:
trial_contents = f'{header}repos:\n{rest}'
yaml_load(trial_contents)
contents = trial_contents
except yaml.YAMLError:
contents = f'{header}repos:\n{textwrap.indent(rest, " " * 4)}'
return contents
def _migrate_sha_to_rev(contents: str) -> str:
return re.sub(r'(\n\s+)sha:', r'\1rev:', contents)
def migrate_config(config_file: str, quiet: bool = False) -> int:
with open(config_file) as f:
orig_contents = contents = f.read()
contents = _migrate_map(contents)
contents = _migrate_sha_to_rev(contents)
if contents != orig_contents:
with open(config_file, 'w') as f:
f.write(contents)
print('Configuration has been migrated.')
elif not quiet:
print('Configuration is already migrated.')
return 0

View file

@ -0,0 +1,421 @@
import argparse
import contextlib
import functools
import logging
import os
import re
import subprocess
import time
import unicodedata
from typing import Any
from typing import Collection
from typing import Dict
from typing import List
from typing import MutableMapping
from typing import Sequence
from typing import Set
from typing import Tuple
from identify.identify import tags_from_path
from pre_commit import color
from pre_commit import git
from pre_commit import output
from pre_commit.clientlib import load_config
from pre_commit.hook import Hook
from pre_commit.languages.all import languages
from pre_commit.repository import all_hooks
from pre_commit.repository import install_hook_envs
from pre_commit.staged_files_only import staged_files_only
from pre_commit.store import Store
from pre_commit.util import cmd_output_b
logger = logging.getLogger('pre_commit')
def _len_cjk(msg: str) -> int:
widths = {'A': 1, 'F': 2, 'H': 1, 'N': 1, 'Na': 1, 'W': 2}
return sum(widths[unicodedata.east_asian_width(c)] for c in msg)
def _start_msg(*, start: str, cols: int, end_len: int) -> str:
dots = '.' * (cols - _len_cjk(start) - end_len - 1)
return f'{start}{dots}'
def _full_msg(
*,
start: str,
cols: int,
end_msg: str,
end_color: str,
use_color: bool,
postfix: str = '',
) -> str:
dots = '.' * (cols - _len_cjk(start) - len(postfix) - len(end_msg) - 1)
end = color.format_color(end_msg, end_color, use_color)
return f'{start}{dots}{postfix}{end}\n'
def filter_by_include_exclude(
names: Collection[str],
include: str,
exclude: str,
) -> List[str]:
include_re, exclude_re = re.compile(include), re.compile(exclude)
return [
filename for filename in names
if include_re.search(filename)
if not exclude_re.search(filename)
]
class Classifier:
def __init__(self, filenames: Collection[str]) -> None:
self.filenames = [f for f in filenames if os.path.lexists(f)]
@functools.lru_cache(maxsize=None)
def _types_for_file(self, filename: str) -> Set[str]:
return tags_from_path(filename)
def by_types(
self,
names: Sequence[str],
types: Collection[str],
types_or: Collection[str],
exclude_types: Collection[str],
) -> List[str]:
types = frozenset(types)
types_or = frozenset(types_or)
exclude_types = frozenset(exclude_types)
ret = []
for filename in names:
tags = self._types_for_file(filename)
if (
tags >= types and
(not types_or or tags & types_or) and
not tags & exclude_types
):
ret.append(filename)
return ret
def filenames_for_hook(self, hook: Hook) -> Tuple[str, ...]:
names = self.filenames
names = filter_by_include_exclude(names, hook.files, hook.exclude)
names = self.by_types(
names,
hook.types,
hook.types_or,
hook.exclude_types,
)
return tuple(names)
@classmethod
def from_config(
cls,
filenames: Collection[str],
include: str,
exclude: str,
) -> 'Classifier':
# on windows we normalize all filenames to use forward slashes
# this makes it easier to filter using the `files:` regex
# this also makes improperly quoted shell-based hooks work better
# see #1173
if os.altsep == '/' and os.sep == '\\':
filenames = [f.replace(os.sep, os.altsep) for f in filenames]
filenames = filter_by_include_exclude(filenames, include, exclude)
return Classifier(filenames)
def _get_skips(environ: MutableMapping[str, str]) -> Set[str]:
skips = environ.get('SKIP', '')
return {skip.strip() for skip in skips.split(',') if skip.strip()}
SKIPPED = 'Skipped'
NO_FILES = '(no files to check)'
def _subtle_line(s: str, use_color: bool) -> None:
output.write_line(color.format_color(s, color.SUBTLE, use_color))
def _run_single_hook(
classifier: Classifier,
hook: Hook,
skips: Set[str],
cols: int,
diff_before: bytes,
verbose: bool,
use_color: bool,
) -> Tuple[bool, bytes]:
filenames = classifier.filenames_for_hook(hook)
if hook.id in skips or hook.alias in skips:
output.write(
_full_msg(
start=hook.name,
end_msg=SKIPPED,
end_color=color.YELLOW,
use_color=use_color,
cols=cols,
),
)
duration = None
retcode = 0
diff_after = diff_before
files_modified = False
out = b''
elif not filenames and not hook.always_run:
output.write(
_full_msg(
start=hook.name,
postfix=NO_FILES,
end_msg=SKIPPED,
end_color=color.TURQUOISE,
use_color=use_color,
cols=cols,
),
)
duration = None
retcode = 0
diff_after = diff_before
files_modified = False
out = b''
else:
# print hook and dots first in case the hook takes a while to run
output.write(_start_msg(start=hook.name, end_len=6, cols=cols))
if not hook.pass_filenames:
filenames = ()
time_before = time.time()
language = languages[hook.language]
retcode, out = language.run_hook(hook, filenames, use_color)
duration = round(time.time() - time_before, 2) or 0
diff_after = _get_diff()
# if the hook makes changes, fail the commit
files_modified = diff_before != diff_after
if retcode or files_modified:
print_color = color.RED
status = 'Failed'
else:
print_color = color.GREEN
status = 'Passed'
output.write_line(color.format_color(status, print_color, use_color))
if verbose or hook.verbose or retcode or files_modified:
_subtle_line(f'- hook id: {hook.id}', use_color)
if (verbose or hook.verbose) and duration is not None:
_subtle_line(f'- duration: {duration}s', use_color)
if retcode:
_subtle_line(f'- exit code: {retcode}', use_color)
# Print a message if failing due to file modifications
if files_modified:
_subtle_line('- files were modified by this hook', use_color)
if out.strip():
output.write_line()
output.write_line_b(out.strip(), logfile_name=hook.log_file)
output.write_line()
return files_modified or bool(retcode), diff_after
def _compute_cols(hooks: Sequence[Hook]) -> int:
"""Compute the number of columns to display hook messages. The widest
that will be displayed is in the no files skipped case:
Hook name...(no files to check) Skipped
"""
if hooks:
name_len = max(_len_cjk(hook.name) for hook in hooks)
else:
name_len = 0
cols = name_len + 3 + len(NO_FILES) + 1 + len(SKIPPED)
return max(cols, 80)
def _all_filenames(args: argparse.Namespace) -> Collection[str]:
# these hooks do not operate on files
if args.hook_stage in {
'post-checkout', 'post-commit', 'post-merge', 'post-rewrite',
}:
return ()
elif args.hook_stage in {'prepare-commit-msg', 'commit-msg'}:
return (args.commit_msg_filename,)
elif args.from_ref and args.to_ref:
return git.get_changed_files(args.from_ref, args.to_ref)
elif args.files:
return args.files
elif args.all_files:
return git.get_all_files()
elif git.is_in_merge_conflict():
return git.get_conflicted_files()
else:
return git.get_staged_files()
def _get_diff() -> bytes:
_, out, _ = cmd_output_b(
'git', 'diff', '--no-ext-diff', '--ignore-submodules', retcode=None,
)
return out
def _run_hooks(
config: Dict[str, Any],
hooks: Sequence[Hook],
skips: Set[str],
args: argparse.Namespace,
) -> int:
"""Actually run the hooks."""
cols = _compute_cols(hooks)
classifier = Classifier.from_config(
_all_filenames(args), config['files'], config['exclude'],
)
retval = 0
prior_diff = _get_diff()
for hook in hooks:
current_retval, prior_diff = _run_single_hook(
classifier, hook, skips, cols, prior_diff,
verbose=args.verbose, use_color=args.color,
)
retval |= current_retval
if retval and (config['fail_fast'] or hook.fail_fast):
break
if retval and args.show_diff_on_failure and prior_diff:
if args.all_files:
output.write_line(
'pre-commit hook(s) made changes.\n'
'If you are seeing this message in CI, '
'reproduce locally with: `pre-commit run --all-files`.\n'
'To run `pre-commit` as part of git workflow, use '
'`pre-commit install`.',
)
output.write_line('All changes made by hooks:')
# args.color is a boolean.
# See user_color function in color.py
git_color_opt = 'always' if args.color else 'never'
subprocess.call((
'git', '--no-pager', 'diff', '--no-ext-diff',
f'--color={git_color_opt}',
))
return retval
def _has_unmerged_paths() -> bool:
_, stdout, _ = cmd_output_b('git', 'ls-files', '--unmerged')
return bool(stdout.strip())
def _has_unstaged_config(config_file: str) -> bool:
retcode, _, _ = cmd_output_b(
'git', 'diff', '--no-ext-diff', '--exit-code', config_file,
retcode=None,
)
# be explicit, other git errors don't mean it has an unstaged config.
return retcode == 1
def run(
config_file: str,
store: Store,
args: argparse.Namespace,
environ: MutableMapping[str, str] = os.environ,
) -> int:
stash = not args.all_files and not args.files
# Check if we have unresolved merge conflict files and fail fast.
if _has_unmerged_paths():
logger.error('Unmerged files. Resolve before committing.')
return 1
if bool(args.from_ref) != bool(args.to_ref):
logger.error('Specify both --from-ref and --to-ref.')
return 1
if stash and _has_unstaged_config(config_file):
logger.error(
f'Your pre-commit configuration is unstaged.\n'
f'`git add {config_file}` to fix this.',
)
return 1
if (
args.hook_stage in {'prepare-commit-msg', 'commit-msg'} and
not args.commit_msg_filename
):
logger.error(
f'`--commit-msg-filename` is required for '
f'`--hook-stage {args.hook_stage}`',
)
return 1
# prevent recursive post-checkout hooks (#1418)
if (
args.hook_stage == 'post-checkout' and
environ.get('_PRE_COMMIT_SKIP_POST_CHECKOUT')
):
return 0
# Expose from-ref / to-ref as environment variables for hooks to consume
if args.from_ref and args.to_ref:
# legacy names
environ['PRE_COMMIT_ORIGIN'] = args.from_ref
environ['PRE_COMMIT_SOURCE'] = args.to_ref
# new names
environ['PRE_COMMIT_FROM_REF'] = args.from_ref
environ['PRE_COMMIT_TO_REF'] = args.to_ref
if (
args.remote_name and args.remote_url and
args.remote_branch and args.local_branch
):
environ['PRE_COMMIT_LOCAL_BRANCH'] = args.local_branch
environ['PRE_COMMIT_REMOTE_BRANCH'] = args.remote_branch
environ['PRE_COMMIT_REMOTE_NAME'] = args.remote_name
environ['PRE_COMMIT_REMOTE_URL'] = args.remote_url
if args.checkout_type:
environ['PRE_COMMIT_CHECKOUT_TYPE'] = args.checkout_type
if args.is_squash_merge:
environ['PRE_COMMIT_IS_SQUASH_MERGE'] = args.is_squash_merge
if args.rewrite_command:
environ['PRE_COMMIT_REWRITE_COMMAND'] = args.rewrite_command
# Set pre_commit flag
environ['PRE_COMMIT'] = '1'
with contextlib.ExitStack() as exit_stack:
if stash:
exit_stack.enter_context(staged_files_only(store.directory))
config = load_config(config_file)
hooks = [
hook
for hook in all_hooks(config, store)
if not args.hook or hook.id == args.hook or hook.alias == args.hook
if args.hook_stage in hook.stages
]
if args.hook and not hooks:
output.write_line(
f'No hook with id `{args.hook}` in stage `{args.hook_stage}`',
)
return 1
skips = _get_skips(environ)
to_install = [hook for hook in hooks if hook.id not in skips]
install_hook_envs(to_install, store)
return _run_hooks(config, hooks, skips, args)
# https://github.com/python/mypy/issues/7726
raise AssertionError('unreachable')

View file

@ -0,0 +1,21 @@
# TODO: maybe `git ls-remote git://github.com/pre-commit/pre-commit-hooks` to
# determine the latest revision? This adds ~200ms from my tests (and is
# significantly faster than https:// or http://). For now, periodically
# manually updating the revision is fine.
SAMPLE_CONFIG = '''\
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.2.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
'''
def sample_config() -> int:
print(SAMPLE_CONFIG, end='')
return 0

View file

@ -0,0 +1,77 @@
import argparse
import logging
import os.path
from typing import Optional
from typing import Tuple
import pre_commit.constants as C
from pre_commit import git
from pre_commit import output
from pre_commit.clientlib import load_manifest
from pre_commit.commands.run import run
from pre_commit.store import Store
from pre_commit.util import cmd_output_b
from pre_commit.util import tmpdir
from pre_commit.util import yaml_dump
from pre_commit.xargs import xargs
logger = logging.getLogger(__name__)
def _repo_ref(tmpdir: str, repo: str, ref: Optional[str]) -> Tuple[str, str]:
# if `ref` is explicitly passed, use it
if ref is not None:
return repo, ref
ref = git.head_rev(repo)
# if it exists on disk, we'll try and clone it with the local changes
if os.path.exists(repo) and git.has_diff('HEAD', repo=repo):
logger.warning('Creating temporary repo with uncommitted changes...')
shadow = os.path.join(tmpdir, 'shadow-repo')
cmd_output_b('git', 'clone', repo, shadow)
cmd_output_b('git', 'checkout', ref, '-b', '_pc_tmp', cwd=shadow)
idx = git.git_path('index', repo=shadow)
objs = git.git_path('objects', repo=shadow)
env = dict(os.environ, GIT_INDEX_FILE=idx, GIT_OBJECT_DIRECTORY=objs)
staged_files = git.get_staged_files(cwd=repo)
if staged_files:
xargs(('git', 'add', '--'), staged_files, cwd=repo, env=env)
cmd_output_b('git', 'add', '-u', cwd=repo, env=env)
git.commit(repo=shadow)
return shadow, git.head_rev(shadow)
else:
return repo, ref
def try_repo(args: argparse.Namespace) -> int:
with tmpdir() as tempdir:
repo, ref = _repo_ref(tempdir, args.repo, args.ref)
store = Store(tempdir)
if args.hook:
hooks = [{'id': args.hook}]
else:
repo_path = store.clone(repo, ref)
manifest = load_manifest(os.path.join(repo_path, C.MANIFEST_FILE))
manifest = sorted(manifest, key=lambda hook: hook['id'])
hooks = [{'id': hook['id']} for hook in manifest]
config = {'repos': [{'repo': repo, 'rev': ref, 'hooks': hooks}]}
config_s = yaml_dump(config)
config_filename = os.path.join(tempdir, C.CONFIG_FILE)
with open(config_filename, 'w') as cfg:
cfg.write(config_s)
output.write_line('=' * 79)
output.write_line('Using config:')
output.write_line('=' * 79)
output.write(config_s)
output.write_line('=' * 79)
return run(config_filename, store, args)

View file

@ -0,0 +1,25 @@
import sys
if sys.version_info >= (3, 8): # pragma: >=3.8 cover
import importlib.metadata as importlib_metadata
else: # pragma: <3.8 cover
import importlib_metadata
CONFIG_FILE = '.pre-commit-config.yaml'
MANIFEST_FILE = '.pre-commit-hooks.yaml'
# Bump when installation changes in a backwards / forwards incompatible way
INSTALLED_STATE_VERSION = '1'
# Bump when modifying `empty_template`
LOCAL_REPO_VERSION = '1'
VERSION = importlib_metadata.version('pre_commit')
# `manual` is not invoked by any installed git hook. See #719
STAGES = (
'commit', 'merge-commit', 'prepare-commit-msg', 'commit-msg',
'post-commit', 'manual', 'post-checkout', 'push', 'post-merge',
'post-rewrite',
)
DEFAULT = 'default'

View file

@ -0,0 +1,62 @@
import contextlib
import enum
import os
from typing import Generator
from typing import MutableMapping
from typing import NamedTuple
from typing import Optional
from typing import Tuple
from typing import Union
_Unset = enum.Enum('_Unset', 'UNSET')
UNSET = _Unset.UNSET
class Var(NamedTuple):
name: str
default: str = ''
SubstitutionT = Tuple[Union[str, Var], ...]
ValueT = Union[str, _Unset, SubstitutionT]
PatchesT = Tuple[Tuple[str, ValueT], ...]
def format_env(parts: SubstitutionT, env: MutableMapping[str, str]) -> str:
return ''.join(
env.get(part.name, part.default) if isinstance(part, Var) else part
for part in parts
)
@contextlib.contextmanager
def envcontext(
patch: PatchesT,
_env: Optional[MutableMapping[str, str]] = None,
) -> Generator[None, None, None]:
"""In this context, `os.environ` is modified according to `patch`.
`patch` is an iterable of 2-tuples (key, value):
`key`: string
`value`:
- string: `environ[key] == value` inside the context.
- UNSET: `key not in environ` inside the context.
- template: A template is a tuple of strings and Var which will be
replaced with the previous environment
"""
env = os.environ if _env is None else _env
before = dict(env)
for k, v in patch:
if v is UNSET:
env.pop(k, None)
elif isinstance(v, tuple):
env[k] = format_env(v, before)
else:
env[k] = v
try:
yield
finally:
env.clear()
env.update(before)

View file

@ -0,0 +1,78 @@
import contextlib
import functools
import os.path
import sys
import traceback
from typing import Generator
import pre_commit.constants as C
from pre_commit import output
from pre_commit.errors import FatalError
from pre_commit.store import Store
from pre_commit.util import cmd_output_b
from pre_commit.util import force_bytes
def _log_and_exit(
msg: str,
ret_code: int,
exc: BaseException,
formatted: str,
) -> None:
error_msg = f'{msg}: {type(exc).__name__}: '.encode() + force_bytes(exc)
output.write_line_b(error_msg)
_, git_version_b, _ = cmd_output_b('git', '--version', retcode=None)
git_version = git_version_b.decode(errors='backslashreplace').rstrip()
storedir = Store().directory
log_path = os.path.join(storedir, 'pre-commit.log')
with contextlib.ExitStack() as ctx:
if os.access(storedir, os.W_OK):
output.write_line(f'Check the log at {log_path}')
log = ctx.enter_context(open(log_path, 'wb'))
else: # pragma: win32 no cover
output.write_line(f'Failed to write to log at {log_path}')
log = sys.stdout.buffer
_log_line = functools.partial(output.write_line, stream=log)
_log_line_b = functools.partial(output.write_line_b, stream=log)
_log_line('### version information')
_log_line()
_log_line('```')
_log_line(f'pre-commit version: {C.VERSION}')
_log_line(f'git --version: {git_version}')
_log_line('sys.version:')
for line in sys.version.splitlines():
_log_line(f' {line}')
_log_line(f'sys.executable: {sys.executable}')
_log_line(f'os.name: {os.name}')
_log_line(f'sys.platform: {sys.platform}')
_log_line('```')
_log_line()
_log_line('### error information')
_log_line()
_log_line('```')
_log_line_b(error_msg)
_log_line('```')
_log_line()
_log_line('```')
_log_line(formatted.rstrip())
_log_line('```')
raise SystemExit(ret_code)
@contextlib.contextmanager
def error_handler() -> Generator[None, None, None]:
try:
yield
except (Exception, KeyboardInterrupt) as e:
if isinstance(e, FatalError):
msg, ret_code = 'An error has occurred', 1
elif isinstance(e, KeyboardInterrupt):
msg, ret_code = 'Interrupted (^C)', 130
else:
msg, ret_code = 'An unexpected error has occurred', 3
_log_and_exit(msg, ret_code, e, traceback.format_exc())

View file

@ -0,0 +1,2 @@
class FatalError(RuntimeError):
pass

View file

@ -0,0 +1,76 @@
import contextlib
import errno
import sys
from typing import Callable
from typing import Generator
if sys.platform == 'win32': # pragma: no cover (windows)
import msvcrt
# https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/locking
# on windows we lock "regions" of files, we don't care about the actual
# byte region so we'll just pick *some* number here.
_region = 0xffff
@contextlib.contextmanager
def _locked(
fileno: int,
blocked_cb: Callable[[], None],
) -> Generator[None, None, None]:
try:
# TODO: https://github.com/python/typeshed/pull/3607
msvcrt.locking(fileno, msvcrt.LK_NBLCK, _region)
except OSError:
blocked_cb()
while True:
try:
# TODO: https://github.com/python/typeshed/pull/3607
msvcrt.locking(fileno, msvcrt.LK_LOCK, _region)
except OSError as e:
# Locking violation. Returned when the _LK_LOCK or _LK_RLCK
# flag is specified and the file cannot be locked after 10
# attempts.
if e.errno != errno.EDEADLOCK:
raise
else:
break
try:
yield
finally:
# From cursory testing, it seems to get unlocked when the file is
# closed so this may not be necessary.
# The documentation however states:
# "Regions should be locked only briefly and should be unlocked
# before closing a file or exiting the program."
# TODO: https://github.com/python/typeshed/pull/3607
msvcrt.locking(fileno, msvcrt.LK_UNLCK, _region)
else: # pragma: win32 no cover
import fcntl
@contextlib.contextmanager
def _locked(
fileno: int,
blocked_cb: Callable[[], None],
) -> Generator[None, None, None]:
try:
fcntl.flock(fileno, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError: # pragma: no cover (tests are single-threaded)
blocked_cb()
fcntl.flock(fileno, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(fileno, fcntl.LOCK_UN)
@contextlib.contextmanager
def lock(
path: str,
blocked_cb: Callable[[], None],
) -> Generator[None, None, None]:
with open(path, 'a+') as f:
with _locked(f.fileno(), blocked_cb):
yield

View file

@ -0,0 +1,232 @@
import logging
import os.path
import sys
from typing import Dict
from typing import List
from typing import MutableMapping
from typing import Optional
from typing import Set
from pre_commit.errors import FatalError
from pre_commit.util import CalledProcessError
from pre_commit.util import cmd_output
from pre_commit.util import cmd_output_b
logger = logging.getLogger(__name__)
# see #2046
NO_FS_MONITOR = ('-c', 'core.useBuiltinFSMonitor=false')
def zsplit(s: str) -> List[str]:
s = s.strip('\0')
if s:
return s.split('\0')
else:
return []
def no_git_env(
_env: Optional[MutableMapping[str, str]] = None,
) -> Dict[str, str]:
# Too many bugs dealing with environment variables and GIT:
# https://github.com/pre-commit/pre-commit/issues/300
# In git 2.6.3 (maybe others), git exports GIT_WORK_TREE while running
# pre-commit hooks
# In git 1.9.1 (maybe others), git exports GIT_DIR and GIT_INDEX_FILE
# while running pre-commit hooks in submodules.
# GIT_DIR: Causes git clone to clone wrong thing
# GIT_INDEX_FILE: Causes 'error invalid object ...' during commit
_env = _env if _env is not None else os.environ
return {
k: v for k, v in _env.items()
if not k.startswith('GIT_') or
k.startswith(('GIT_CONFIG_KEY_', 'GIT_CONFIG_VALUE_')) or
k in {
'GIT_EXEC_PATH', 'GIT_SSH', 'GIT_SSH_COMMAND', 'GIT_SSL_CAINFO',
'GIT_SSL_NO_VERIFY', 'GIT_CONFIG_COUNT',
}
}
def get_root() -> str:
# Git 2.25 introduced a change to "rev-parse --show-toplevel" that exposed
# underlying volumes for Windows drives mapped with SUBST. We use
# "rev-parse --show-cdup" to get the appropriate path, but must perform
# an extra check to see if we are in the .git directory.
try:
root = os.path.abspath(
cmd_output('git', 'rev-parse', '--show-cdup')[1].strip(),
)
git_dir = os.path.abspath(get_git_dir())
except CalledProcessError:
raise FatalError(
'git failed. Is it installed, and are you in a Git repository '
'directory?',
)
if os.path.samefile(root, git_dir):
raise FatalError(
'git toplevel unexpectedly empty! make sure you are not '
'inside the `.git` directory of your repository.',
)
return root
def get_git_dir(git_root: str = '.') -> str:
opts = ('--git-common-dir', '--git-dir')
_, out, _ = cmd_output('git', 'rev-parse', *opts, cwd=git_root)
for line, opt in zip(out.splitlines(), opts):
if line != opt: # pragma: no branch (git < 2.5)
return os.path.normpath(os.path.join(git_root, line))
else:
raise AssertionError('unreachable: no git dir')
def get_remote_url(git_root: str) -> str:
_, out, _ = cmd_output('git', 'config', 'remote.origin.url', cwd=git_root)
return out.strip()
def is_in_merge_conflict() -> bool:
git_dir = get_git_dir('.')
return (
os.path.exists(os.path.join(git_dir, 'MERGE_MSG')) and
os.path.exists(os.path.join(git_dir, 'MERGE_HEAD'))
)
def parse_merge_msg_for_conflicts(merge_msg: bytes) -> List[str]:
# Conflicted files start with tabs
return [
line.lstrip(b'#').strip().decode()
for line in merge_msg.splitlines()
# '#\t' for git 2.4.1
if line.startswith((b'\t', b'#\t'))
]
def get_conflicted_files() -> Set[str]:
logger.info('Checking merge-conflict files only.')
# Need to get the conflicted files from the MERGE_MSG because they could
# have resolved the conflict by choosing one side or the other
with open(os.path.join(get_git_dir('.'), 'MERGE_MSG'), 'rb') as f:
merge_msg = f.read()
merge_conflict_filenames = parse_merge_msg_for_conflicts(merge_msg)
# This will get the rest of the changes made after the merge.
# If they resolved the merge conflict by choosing a mesh of both sides
# this will also include the conflicted files
tree_hash = cmd_output('git', 'write-tree')[1].strip()
merge_diff_filenames = zsplit(
cmd_output(
'git', 'diff', '--name-only', '--no-ext-diff', '-z',
'-m', tree_hash, 'HEAD', 'MERGE_HEAD',
)[1],
)
return set(merge_conflict_filenames) | set(merge_diff_filenames)
def get_staged_files(cwd: Optional[str] = None) -> List[str]:
return zsplit(
cmd_output(
'git', 'diff', '--staged', '--name-only', '--no-ext-diff', '-z',
# Everything except for D
'--diff-filter=ACMRTUXB',
cwd=cwd,
)[1],
)
def intent_to_add_files() -> List[str]:
_, stdout, _ = cmd_output(
'git', 'status', '--ignore-submodules', '--porcelain', '-z',
)
parts = list(reversed(zsplit(stdout)))
intent_to_add = []
while parts:
line = parts.pop()
status, filename = line[:3], line[3:]
if status[0] in {'C', 'R'}: # renames / moves have an additional arg
parts.pop()
if status[1] == 'A':
intent_to_add.append(filename)
return intent_to_add
def get_all_files() -> List[str]:
return zsplit(cmd_output('git', 'ls-files', '-z')[1])
def get_changed_files(old: str, new: str) -> List[str]:
diff_cmd = ('git', 'diff', '--name-only', '--no-ext-diff', '-z')
try:
_, out, _ = cmd_output(*diff_cmd, f'{old}...{new}')
except CalledProcessError: # pragma: no cover (new git)
# on newer git where old and new do not have a merge base git fails
# so we try a full diff (this is what old git did for us!)
_, out, _ = cmd_output(*diff_cmd, f'{old}..{new}')
return zsplit(out)
def head_rev(remote: str) -> str:
_, out, _ = cmd_output('git', 'ls-remote', '--exit-code', remote, 'HEAD')
return out.split()[0]
def has_diff(*args: str, repo: str = '.') -> bool:
cmd = ('git', 'diff', '--quiet', '--no-ext-diff', *args)
return cmd_output_b(*cmd, cwd=repo, retcode=None)[0] == 1
def has_core_hookpaths_set() -> bool:
_, out, _ = cmd_output_b('git', 'config', 'core.hooksPath', retcode=None)
return bool(out.strip())
def init_repo(path: str, remote: str) -> None:
if os.path.isdir(remote):
remote = os.path.abspath(remote)
git = ('git', *NO_FS_MONITOR)
env = no_git_env()
# avoid the user's template so that hooks do not recurse
cmd_output_b(*git, 'init', '--template=', path, env=env)
cmd_output_b(*git, 'remote', 'add', 'origin', remote, cwd=path, env=env)
def commit(repo: str = '.') -> None:
env = no_git_env()
name, email = 'pre-commit', 'asottile+pre-commit@umich.edu'
env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = name
env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = email
cmd = ('git', 'commit', '--no-edit', '--no-gpg-sign', '-n', '-minit')
cmd_output_b(*cmd, cwd=repo, env=env)
def git_path(name: str, repo: str = '.') -> str:
_, out, _ = cmd_output('git', 'rev-parse', '--git-path', name, cwd=repo)
return os.path.join(repo, out.strip())
def check_for_cygwin_mismatch() -> None:
"""See https://github.com/pre-commit/pre-commit/issues/354"""
if sys.platform in ('cygwin', 'win32'): # pragma: no cover (windows)
is_cygwin_python = sys.platform == 'cygwin'
try:
toplevel = get_root()
except FatalError: # skip the check if we're not in a git repo
return
is_cygwin_git = toplevel.startswith('/')
if is_cygwin_python ^ is_cygwin_git:
exe_type = {True: '(cygwin)', False: '(windows)'}
logger.warn(
f'pre-commit has detected a mix of cygwin python / git\n'
f'This combination is not supported, it is likely you will '
f'receive an error later in the program.\n'
f'Make sure to use cygwin git+python while using cygwin\n'
f'These can be installed through the cygwin installer.\n'
f' - python {exe_type[is_cygwin_python]}\n'
f' - git {exe_type[is_cygwin_git]}\n',
)

View file

@ -0,0 +1,65 @@
import logging
import shlex
from typing import Any
from typing import Dict
from typing import NamedTuple
from typing import Sequence
from typing import Tuple
from pre_commit.prefix import Prefix
logger = logging.getLogger('pre_commit')
class Hook(NamedTuple):
src: str
prefix: Prefix
id: str
name: str
entry: str
language: str
alias: str
files: str
exclude: str
types: Sequence[str]
types_or: Sequence[str]
exclude_types: Sequence[str]
additional_dependencies: Sequence[str]
args: Sequence[str]
always_run: bool
fail_fast: bool
pass_filenames: bool
description: str
language_version: str
log_file: str
minimum_pre_commit_version: str
require_serial: bool
stages: Sequence[str]
verbose: bool
@property
def cmd(self) -> Tuple[str, ...]:
return (*shlex.split(self.entry), *self.args)
@property
def install_key(self) -> Tuple[Prefix, str, str, Tuple[str, ...]]:
return (
self.prefix,
self.language,
self.language_version,
tuple(self.additional_dependencies),
)
@classmethod
def create(cls, src: str, prefix: Prefix, dct: Dict[str, Any]) -> 'Hook':
# TODO: have cfgv do this (?)
extra_keys = set(dct) - _KEYS
if extra_keys:
logger.warning(
f'Unexpected key(s) present on {src} => {dct["id"]}: '
f'{", ".join(sorted(extra_keys))}',
)
return cls(src=src, prefix=prefix, **{k: dct[k] for k in _KEYS})
_KEYS = frozenset(set(Hook._fields) - {'src', 'prefix'})

View file

@ -0,0 +1,70 @@
from typing import Callable
from typing import NamedTuple
from typing import Optional
from typing import Sequence
from typing import Tuple
from pre_commit.hook import Hook
from pre_commit.languages import conda
from pre_commit.languages import coursier
from pre_commit.languages import dart
from pre_commit.languages import docker
from pre_commit.languages import docker_image
from pre_commit.languages import dotnet
from pre_commit.languages import fail
from pre_commit.languages import golang
from pre_commit.languages import lua
from pre_commit.languages import node
from pre_commit.languages import perl
from pre_commit.languages import pygrep
from pre_commit.languages import python
from pre_commit.languages import r
from pre_commit.languages import ruby
from pre_commit.languages import rust
from pre_commit.languages import script
from pre_commit.languages import swift
from pre_commit.languages import system
from pre_commit.prefix import Prefix
class Language(NamedTuple):
name: str
# Use `None` for no installation / environment
ENVIRONMENT_DIR: Optional[str]
# return a value to replace `'default` for `language_version`
get_default_version: Callable[[], str]
# return whether the environment is healthy (or should be rebuilt)
healthy: Callable[[Prefix, str], bool]
# install a repository for the given language and language_version
install_environment: Callable[[Prefix, str, Sequence[str]], None]
# execute a hook and return the exit code and output
run_hook: 'Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]]'
# TODO: back to modules + Protocol: https://github.com/python/mypy/issues/5018
languages = {
# BEGIN GENERATED (testing/gen-languages-all)
'conda': Language(name='conda', ENVIRONMENT_DIR=conda.ENVIRONMENT_DIR, get_default_version=conda.get_default_version, healthy=conda.healthy, install_environment=conda.install_environment, run_hook=conda.run_hook), # noqa: E501
'coursier': Language(name='coursier', ENVIRONMENT_DIR=coursier.ENVIRONMENT_DIR, get_default_version=coursier.get_default_version, healthy=coursier.healthy, install_environment=coursier.install_environment, run_hook=coursier.run_hook), # noqa: E501
'dart': Language(name='dart', ENVIRONMENT_DIR=dart.ENVIRONMENT_DIR, get_default_version=dart.get_default_version, healthy=dart.healthy, install_environment=dart.install_environment, run_hook=dart.run_hook), # noqa: E501
'docker': Language(name='docker', ENVIRONMENT_DIR=docker.ENVIRONMENT_DIR, get_default_version=docker.get_default_version, healthy=docker.healthy, install_environment=docker.install_environment, run_hook=docker.run_hook), # noqa: E501
'docker_image': Language(name='docker_image', ENVIRONMENT_DIR=docker_image.ENVIRONMENT_DIR, get_default_version=docker_image.get_default_version, healthy=docker_image.healthy, install_environment=docker_image.install_environment, run_hook=docker_image.run_hook), # noqa: E501
'dotnet': Language(name='dotnet', ENVIRONMENT_DIR=dotnet.ENVIRONMENT_DIR, get_default_version=dotnet.get_default_version, healthy=dotnet.healthy, install_environment=dotnet.install_environment, run_hook=dotnet.run_hook), # noqa: E501
'fail': Language(name='fail', ENVIRONMENT_DIR=fail.ENVIRONMENT_DIR, get_default_version=fail.get_default_version, healthy=fail.healthy, install_environment=fail.install_environment, run_hook=fail.run_hook), # noqa: E501
'golang': Language(name='golang', ENVIRONMENT_DIR=golang.ENVIRONMENT_DIR, get_default_version=golang.get_default_version, healthy=golang.healthy, install_environment=golang.install_environment, run_hook=golang.run_hook), # noqa: E501
'lua': Language(name='lua', ENVIRONMENT_DIR=lua.ENVIRONMENT_DIR, get_default_version=lua.get_default_version, healthy=lua.healthy, install_environment=lua.install_environment, run_hook=lua.run_hook), # noqa: E501
'node': Language(name='node', ENVIRONMENT_DIR=node.ENVIRONMENT_DIR, get_default_version=node.get_default_version, healthy=node.healthy, install_environment=node.install_environment, run_hook=node.run_hook), # noqa: E501
'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook), # noqa: E501
'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook), # noqa: E501
'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook), # noqa: E501
'r': Language(name='r', ENVIRONMENT_DIR=r.ENVIRONMENT_DIR, get_default_version=r.get_default_version, healthy=r.healthy, install_environment=r.install_environment, run_hook=r.run_hook), # noqa: E501
'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook), # noqa: E501
'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook), # noqa: E501
'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook), # noqa: E501
'swift': Language(name='swift', ENVIRONMENT_DIR=swift.ENVIRONMENT_DIR, get_default_version=swift.get_default_version, healthy=swift.healthy, install_environment=swift.install_environment, run_hook=swift.run_hook), # noqa: E501
'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook), # noqa: E501
# END GENERATED
}
# TODO: fully deprecate `python_venv`
languages['python_venv'] = languages['python']
all_languages = sorted(languages)

View file

@ -0,0 +1,95 @@
import contextlib
import os
from typing import Generator
from typing import Sequence
from typing import Tuple
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import SubstitutionT
from pre_commit.envcontext import UNSET
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output_b
ENVIRONMENT_DIR = 'conda'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def get_env_patch(env: str) -> PatchesT:
# On non-windows systems executable live in $CONDA_PREFIX/bin, on Windows
# they can be in $CONDA_PREFIX/bin, $CONDA_PREFIX/Library/bin,
# $CONDA_PREFIX/Scripts and $CONDA_PREFIX. Whereas the latter only
# seems to be used for python.exe.
path: SubstitutionT = (os.path.join(env, 'bin'), os.pathsep, Var('PATH'))
if os.name == 'nt': # pragma: no cover (platform specific)
path = (env, os.pathsep, *path)
path = (os.path.join(env, 'Scripts'), os.pathsep, *path)
path = (os.path.join(env, 'Library', 'bin'), os.pathsep, *path)
return (
('PYTHONHOME', UNSET),
('VIRTUAL_ENV', UNSET),
('CONDA_PREFIX', env),
('PATH', path),
)
@contextlib.contextmanager
def in_env(
prefix: Prefix,
language_version: str,
) -> Generator[None, None, None]:
directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
envdir = prefix.path(directory)
with envcontext(get_env_patch(envdir)):
yield
def _conda_exe() -> str:
if os.environ.get('PRE_COMMIT_USE_MICROMAMBA'):
return 'micromamba'
elif os.environ.get('PRE_COMMIT_USE_MAMBA'):
return 'mamba'
else:
return 'conda'
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('conda', version)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
conda_exe = _conda_exe()
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
cmd_output_b(
conda_exe, 'env', 'create', '-p', env_dir, '--file',
'environment.yml', cwd=prefix.prefix_dir,
)
if additional_dependencies:
cmd_output_b(
conda_exe, 'install', '-p', env_dir, *additional_dependencies,
cwd=prefix.prefix_dir,
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
# TODO: Some rare commands need to be run using `conda run` but mostly we
# can run them without which is much quicker and produces a better
# output.
# cmd = ('conda', 'run', '-p', env_dir) + hook.cmd
with in_env(hook.prefix, hook.language_version):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,71 @@
import contextlib
import os
from typing import Generator
from typing import Sequence
from typing import Tuple
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
ENVIRONMENT_DIR = 'coursier'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('coursier', version)
helpers.assert_no_additional_deps('coursier', additional_dependencies)
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
channel = prefix.path('.pre-commit-channel')
with clean_path_on_failure(envdir):
for app_descriptor in os.listdir(channel):
_, app_file = os.path.split(app_descriptor)
app, _ = os.path.splitext(app_file)
helpers.run_setup_cmd(
prefix,
(
'cs',
'install',
'--default-channels=false',
f'--channel={channel}',
app,
f'--dir={envdir}',
),
)
def get_env_patch(target_dir: str) -> PatchesT: # pragma: win32 no cover
return (
('PATH', (target_dir, os.pathsep, Var('PATH'))),
)
@contextlib.contextmanager
def in_env(
prefix: Prefix,
) -> Generator[None, None, None]: # pragma: win32 no cover
target_dir = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, get_default_version()),
)
with envcontext(get_env_patch(target_dir)):
yield
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]: # pragma: win32 no cover
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,109 @@
import contextlib
import os.path
import shutil
import tempfile
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import win_exe
from pre_commit.util import yaml_load
ENVIRONMENT_DIR = 'dartenv'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def get_env_patch(venv: str) -> PatchesT:
return (
('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
)
@contextlib.contextmanager
def in_env(prefix: Prefix) -> Generator[None, None, None]:
directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT)
envdir = prefix.path(directory)
with envcontext(get_env_patch(envdir)):
yield
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('dart', version)
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
bin_dir = os.path.join(envdir, 'bin')
def _install_dir(prefix_p: Prefix, pub_cache: str) -> None:
dart_env = {**os.environ, 'PUB_CACHE': pub_cache}
with open(prefix_p.path('pubspec.yaml')) as f:
pubspec_contents = yaml_load(f)
helpers.run_setup_cmd(prefix_p, ('dart', 'pub', 'get'), env=dart_env)
for executable in pubspec_contents['executables']:
helpers.run_setup_cmd(
prefix_p,
(
'dart', 'compile', 'exe',
'--output', os.path.join(bin_dir, win_exe(executable)),
prefix_p.path('bin', f'{executable}.dart'),
),
env=dart_env,
)
with clean_path_on_failure(envdir):
os.makedirs(bin_dir)
with tempfile.TemporaryDirectory() as tmp:
_install_dir(prefix, tmp)
for dep_s in additional_dependencies:
with tempfile.TemporaryDirectory() as dep_tmp:
dep, _, version = dep_s.partition(':')
if version:
dep_cmd: Tuple[str, ...] = (dep, '--version', version)
else:
dep_cmd = (dep,)
helpers.run_setup_cmd(
prefix,
('dart', 'pub', 'cache', 'add', *dep_cmd),
env={**os.environ, 'PUB_CACHE': dep_tmp},
)
# try and find the 'pubspec.yaml' that just got added
for root, _, filenames in os.walk(dep_tmp):
if 'pubspec.yaml' in filenames:
with tempfile.TemporaryDirectory() as copied:
pkg = os.path.join(copied, 'pkg')
shutil.copytree(root, pkg)
_install_dir(Prefix(pkg), dep_tmp)
break
else:
raise AssertionError(
f'could not find pubspec.yaml for {dep_s}',
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,141 @@
import hashlib
import json
import os
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import CalledProcessError
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output_b
ENVIRONMENT_DIR = 'docker'
PRE_COMMIT_LABEL = 'PRE_COMMIT'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def _is_in_docker() -> bool:
try:
with open('/proc/1/cgroup', 'rb') as f:
return b'docker' in f.read()
except FileNotFoundError:
return False
def _get_container_id() -> str:
# It's assumed that we already check /proc/1/cgroup in _is_in_docker. The
# cpuset cgroup controller existed since cgroups were introduced so this
# way of getting the container ID is pretty reliable.
with open('/proc/1/cgroup', 'rb') as f:
for line in f.readlines():
if line.split(b':')[1] == b'cpuset':
return os.path.basename(line.split(b':')[2]).strip().decode()
raise RuntimeError('Failed to find the container ID in /proc/1/cgroup.')
def _get_docker_path(path: str) -> str:
if not _is_in_docker():
return path
container_id = _get_container_id()
try:
_, out, _ = cmd_output_b('docker', 'inspect', container_id)
except CalledProcessError:
# self-container was not visible from here (perhaps docker-in-docker)
return path
container, = json.loads(out)
for mount in container['Mounts']:
src_path = mount['Source']
to_path = mount['Destination']
if os.path.commonpath((path, to_path)) == to_path:
# So there is something in common,
# and we can proceed remapping it
return path.replace(to_path, src_path)
# we're in Docker, but the path is not mounted, cannot really do anything,
# so fall back to original path
return path
def md5(s: str) -> str: # pragma: win32 no cover
return hashlib.md5(s.encode()).hexdigest()
def docker_tag(prefix: Prefix) -> str: # pragma: win32 no cover
md5sum = md5(os.path.basename(prefix.prefix_dir)).lower()
return f'pre-commit-{md5sum}'
def build_docker_image(
prefix: Prefix,
*,
pull: bool,
) -> None: # pragma: win32 no cover
cmd: Tuple[str, ...] = (
'docker', 'build',
'--tag', docker_tag(prefix),
'--label', PRE_COMMIT_LABEL,
)
if pull:
cmd += ('--pull',)
# This must come last for old versions of docker. See #477
cmd += ('.',)
helpers.run_setup_cmd(prefix, cmd)
def install_environment(
prefix: Prefix, version: str, additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup its state files on failure
with clean_path_on_failure(directory):
build_docker_image(prefix, pull=True)
os.mkdir(directory)
def get_docker_user() -> Tuple[str, ...]: # pragma: win32 no cover
try:
return ('-u', f'{os.getuid()}:{os.getgid()}')
except AttributeError:
return ()
def docker_cmd() -> Tuple[str, ...]: # pragma: win32 no cover
return (
'docker', 'run',
'--rm',
*get_docker_user(),
# https://docs.docker.com/engine/reference/commandline/run/#mount-volumes-from-container-volumes-from
# The `Z` option tells Docker to label the content with a private
# unshared label. Only the current container can use a private volume.
'-v', f'{_get_docker_path(os.getcwd())}:/src:rw,Z',
'--workdir', '/src',
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]: # pragma: win32 no cover
# Rebuild the docker image in case it has gone missing, as many people do
# automated cleanup of docker images.
build_docker_image(hook.prefix, pull=False)
entry_exe, *cmd_rest = hook.cmd
entry_tag = ('--entrypoint', entry_exe, docker_tag(hook.prefix))
cmd = (*docker_cmd(), *entry_tag, *cmd_rest)
return helpers.run_xargs(hook, cmd, file_args, color=color)

View file

@ -0,0 +1,20 @@
from typing import Sequence
from typing import Tuple
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.languages.docker import docker_cmd
ENVIRONMENT_DIR = None
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
install_environment = helpers.no_install
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]: # pragma: win32 no cover
cmd = docker_cmd() + hook.cmd
return helpers.run_xargs(hook, cmd, file_args, color=color)

View file

@ -0,0 +1,89 @@
import contextlib
import os.path
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
ENVIRONMENT_DIR = 'dotnetenv'
BIN_DIR = 'bin'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def get_env_patch(venv: str) -> PatchesT:
return (
('PATH', (os.path.join(venv, BIN_DIR), os.pathsep, Var('PATH'))),
)
@contextlib.contextmanager
def in_env(prefix: Prefix) -> Generator[None, None, None]:
directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT)
envdir = prefix.path(directory)
with envcontext(get_env_patch(envdir)):
yield
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('dotnet', version)
helpers.assert_no_additional_deps('dotnet', additional_dependencies)
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
with clean_path_on_failure(envdir):
build_dir = 'pre-commit-build'
# Build & pack nupkg file
helpers.run_setup_cmd(
prefix,
(
'dotnet', 'pack',
'--configuration', 'Release',
'--output', build_dir,
),
)
# Determine tool from the packaged file <tool_name>.<version>.nupkg
build_outputs = os.listdir(os.path.join(prefix.prefix_dir, build_dir))
if len(build_outputs) != 1:
raise NotImplementedError(
f"Can't handle multiple build outputs. Got {build_outputs}",
)
tool_name = build_outputs[0].split('.')[0]
# Install to bin dir
helpers.run_setup_cmd(
prefix,
(
'dotnet', 'tool', 'install',
'--tool-path', os.path.join(envdir, BIN_DIR),
'--add-source', build_dir,
tool_name,
),
)
# Clean the git dir, ignoring the environment dir
clean_cmd = ('git', 'clean', '-ffxd', '-e', f'{ENVIRONMENT_DIR}-*')
helpers.run_setup_cmd(prefix, clean_cmd)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,20 @@
from typing import Sequence
from typing import Tuple
from pre_commit.hook import Hook
from pre_commit.languages import helpers
ENVIRONMENT_DIR = None
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
install_environment = helpers.no_install
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
out = f'{hook.entry}\n\n'.encode()
out += b'\n'.join(f.encode() for f in file_args) + b'\n'
return 1, out

View file

@ -0,0 +1,100 @@
import contextlib
import os.path
import sys
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit import git
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output
from pre_commit.util import cmd_output_b
from pre_commit.util import rmtree
ENVIRONMENT_DIR = 'golangenv'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def get_env_patch(venv: str) -> PatchesT:
return (
('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
)
@contextlib.contextmanager
def in_env(prefix: Prefix) -> Generator[None, None, None]:
envdir = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
with envcontext(get_env_patch(envdir)):
yield
def guess_go_dir(remote_url: str) -> str:
if remote_url.endswith('.git'):
remote_url = remote_url[:-1 * len('.git')]
looks_like_url = (
not remote_url.startswith('file://') and
('//' in remote_url or '@' in remote_url)
)
remote_url = remote_url.replace(':', '/')
if looks_like_url:
_, _, remote_url = remote_url.rpartition('//')
_, _, remote_url = remote_url.rpartition('@')
return remote_url
else:
return 'unknown_src_dir'
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('golang', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
with clean_path_on_failure(directory):
remote = git.get_remote_url(prefix.prefix_dir)
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
cmd = ('git', 'clone', '--recursive', '.', repo_src_dir)
helpers.run_setup_cmd(prefix, cmd)
if sys.platform == 'cygwin': # pragma: no cover
_, gopath, _ = cmd_output('cygpath', '-w', directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
env.pop('GOBIN', None)
cmd_output_b('go', 'install', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output_b(
'go', 'install', dependency, cwd=repo_src_dir, env=env,
)
# Same some disk space, we don't need these after installation
rmtree(prefix.path(directory, 'src'))
pkgdir = prefix.path(directory, 'pkg')
if os.path.exists(pkgdir): # pragma: no cover (go<1.10)
rmtree(pkgdir)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,136 @@
import multiprocessing
import os
import random
import re
from typing import Any
from typing import List
from typing import Optional
from typing import overload
from typing import Sequence
from typing import Tuple
from typing import TYPE_CHECKING
import pre_commit.constants as C
from pre_commit import parse_shebang
from pre_commit.hook import Hook
from pre_commit.prefix import Prefix
from pre_commit.util import cmd_output_b
from pre_commit.xargs import xargs
if TYPE_CHECKING:
from typing import NoReturn
FIXED_RANDOM_SEED = 1542676187
SHIMS_RE = re.compile(r'[/\\]shims[/\\]')
def exe_exists(exe: str) -> bool:
found = parse_shebang.find_executable(exe)
if found is None: # exe exists
return False
homedir = os.path.expanduser('~')
try:
common: Optional[str] = os.path.commonpath((found, homedir))
except ValueError: # on windows, different drives raises ValueError
common = None
return (
# it is not in a /shims/ directory
not SHIMS_RE.search(found) and
(
# the homedir is / (docker, service user, etc.)
os.path.dirname(homedir) == homedir or
# the exe is not contained in the home directory
common != homedir
)
)
def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...], **kwargs: Any) -> None:
cmd_output_b(*cmd, cwd=prefix.prefix_dir, **kwargs)
@overload
def environment_dir(d: None, language_version: str) -> None: ...
@overload
def environment_dir(d: str, language_version: str) -> str: ...
def environment_dir(d: Optional[str], language_version: str) -> Optional[str]:
if d is None:
return None
else:
return f'{d}-{language_version}'
def assert_version_default(binary: str, version: str) -> None:
if version != C.DEFAULT:
raise AssertionError(
f'For now, pre-commit requires system-installed {binary}',
)
def assert_no_additional_deps(
lang: str,
additional_deps: Sequence[str],
) -> None:
if additional_deps:
raise AssertionError(
f'For now, pre-commit does not support '
f'additional_dependencies for {lang}',
)
def basic_get_default_version() -> str:
return C.DEFAULT
def basic_healthy(prefix: Prefix, language_version: str) -> bool:
return True
def no_install(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> 'NoReturn':
raise AssertionError('This type is not installable')
def target_concurrency(hook: Hook) -> int:
if hook.require_serial or 'PRE_COMMIT_NO_CONCURRENCY' in os.environ:
return 1
else:
# Travis appears to have a bunch of CPUs, but we can't use them all.
if 'TRAVIS' in os.environ:
return 2
else:
try:
return multiprocessing.cpu_count()
except NotImplementedError:
return 1
def _shuffled(seq: Sequence[str]) -> List[str]:
"""Deterministically shuffle"""
fixed_random = random.Random()
fixed_random.seed(FIXED_RANDOM_SEED, version=1)
seq = list(seq)
fixed_random.shuffle(seq)
return seq
def run_xargs(
hook: Hook,
cmd: Tuple[str, ...],
file_args: Sequence[str],
**kwargs: Any,
) -> Tuple[int, bytes]:
# Shuffle the files so that they more evenly fill out the xargs partitions,
# but do it deterministically in case a hook cares about ordering.
file_args = _shuffled(file_args)
kwargs['target_concurrency'] = target_concurrency(hook)
return xargs(cmd, file_args, **kwargs)

View file

@ -0,0 +1,90 @@
import contextlib
import os
import sys
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output
ENVIRONMENT_DIR = 'lua_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def _get_lua_version() -> str: # pragma: win32 no cover
"""Get the Lua version used in file paths."""
_, stdout, _ = cmd_output('luarocks', 'config', '--lua-ver')
return stdout.strip()
def get_env_patch(d: str) -> PatchesT: # pragma: win32 no cover
version = _get_lua_version()
so_ext = 'dll' if sys.platform == 'win32' else 'so'
return (
('PATH', (os.path.join(d, 'bin'), os.pathsep, Var('PATH'))),
(
'LUA_PATH', (
os.path.join(d, 'share', 'lua', version, '?.lua;'),
os.path.join(d, 'share', 'lua', version, '?', 'init.lua;;'),
),
),
(
'LUA_CPATH',
(os.path.join(d, 'lib', 'lua', version, f'?.{so_ext};;'),),
),
)
def _envdir(prefix: Prefix) -> str: # pragma: win32 no cover
directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT)
return prefix.path(directory)
@contextlib.contextmanager # pragma: win32 no cover
def in_env(prefix: Prefix) -> Generator[None, None, None]:
with envcontext(get_env_patch(_envdir(prefix))):
yield
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('lua', version)
envdir = _envdir(prefix)
with clean_path_on_failure(envdir):
with in_env(prefix):
# luarocks doesn't bootstrap a tree prior to installing
# so ensure the directory exists.
os.makedirs(envdir, exist_ok=True)
# Older luarocks (e.g., 2.4.2) expect the rockspec as an arg
for rockspec in prefix.star('.rockspec'):
make_cmd = ('luarocks', '--tree', envdir, 'make', rockspec)
helpers.run_setup_cmd(prefix, make_cmd)
# luarocks can't install multiple packages at once
# so install them individually.
for dependency in additional_dependencies:
cmd = ('luarocks', '--tree', envdir, 'install', dependency)
helpers.run_setup_cmd(prefix, cmd)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]: # pragma: win32 no cover
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,127 @@
import contextlib
import functools
import os
import sys
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import UNSET
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.languages.python import bin_dir
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output
from pre_commit.util import cmd_output_b
from pre_commit.util import rmtree
ENVIRONMENT_DIR = 'node_env'
@functools.lru_cache(maxsize=1)
def get_default_version() -> str:
# nodeenv does not yet support `-n system` on windows
if sys.platform == 'win32':
return C.DEFAULT
# if node is already installed, we can save a bunch of setup time by
# using the installed version
elif all(helpers.exe_exists(exe) for exe in ('node', 'npm')):
return 'system'
else:
return C.DEFAULT
def _envdir(prefix: Prefix, version: str) -> str:
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
return prefix.path(directory)
def get_env_patch(venv: str) -> PatchesT:
if sys.platform == 'cygwin': # pragma: no cover
_, win_venv, _ = cmd_output('cygpath', '-w', venv)
install_prefix = fr'{win_venv.strip()}\bin'
lib_dir = 'lib'
elif sys.platform == 'win32': # pragma: no cover
install_prefix = bin_dir(venv)
lib_dir = 'Scripts'
else: # pragma: win32 no cover
install_prefix = venv
lib_dir = 'lib'
return (
('NODE_VIRTUAL_ENV', venv),
('NPM_CONFIG_PREFIX', install_prefix),
('npm_config_prefix', install_prefix),
('NPM_CONFIG_USERCONFIG', UNSET),
('npm_config_userconfig', UNSET),
('NODE_PATH', os.path.join(venv, lib_dir, 'node_modules')),
('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
)
@contextlib.contextmanager
def in_env(
prefix: Prefix,
language_version: str,
) -> Generator[None, None, None]:
with envcontext(get_env_patch(_envdir(prefix, language_version))):
yield
def healthy(prefix: Prefix, language_version: str) -> bool:
with in_env(prefix, language_version):
retcode, _, _ = cmd_output_b('node', '--version', retcode=None)
return retcode == 0
def install_environment(
prefix: Prefix, version: str, additional_dependencies: Sequence[str],
) -> None:
additional_dependencies = tuple(additional_dependencies)
assert prefix.exists('package.json')
envdir = _envdir(prefix, version)
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
if sys.platform == 'win32': # pragma: no cover
envdir = fr'\\?\{os.path.normpath(envdir)}'
with clean_path_on_failure(envdir):
cmd = [
sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir,
]
if version != C.DEFAULT:
cmd.extend(['-n', version])
cmd_output_b(*cmd)
with in_env(prefix, version):
# https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
# install as if we installed from git
local_install_cmd = (
'npm', 'install', '--dev', '--prod',
'--ignore-prepublish', '--no-progress', '--no-save',
)
helpers.run_setup_cmd(prefix, local_install_cmd)
_, pkg, _ = cmd_output('npm', 'pack', cwd=prefix.prefix_dir)
pkg = prefix.path(pkg.strip())
install = ('npm', 'install', '-g', pkg, *additional_dependencies)
helpers.run_setup_cmd(prefix, install)
# clean these up after installation
if prefix.exists('node_modules'): # pragma: win32 no cover
rmtree(prefix.path('node_modules'))
os.remove(pkg)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix, hook.language_version):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,67 @@
import contextlib
import os
import shlex
from typing import Generator
from typing import Sequence
from typing import Tuple
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
ENVIRONMENT_DIR = 'perl_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def _envdir(prefix: Prefix, version: str) -> str:
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
return prefix.path(directory)
def get_env_patch(venv: str) -> PatchesT:
return (
('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
('PERL5LIB', os.path.join(venv, 'lib', 'perl5')),
('PERL_MB_OPT', f'--install_base {shlex.quote(venv)}'),
(
'PERL_MM_OPT', (
f'INSTALL_BASE={shlex.quote(venv)} '
f'INSTALLSITEMAN1DIR=none INSTALLSITEMAN3DIR=none'
),
),
)
@contextlib.contextmanager
def in_env(
prefix: Prefix,
language_version: str,
) -> Generator[None, None, None]:
with envcontext(get_env_patch(_envdir(prefix, language_version))):
yield
def install_environment(
prefix: Prefix, version: str, additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('perl', version)
with clean_path_on_failure(_envdir(prefix, version)):
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix, ('cpan', '-T', '.', *additional_dependencies),
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix, hook.language_version):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,127 @@
import argparse
import re
import sys
from typing import NamedTuple
from typing import Optional
from typing import Pattern
from typing import Sequence
from typing import Tuple
from pre_commit import output
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.xargs import xargs
ENVIRONMENT_DIR = None
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
install_environment = helpers.no_install
def _process_filename_by_line(pattern: Pattern[bytes], filename: str) -> int:
retv = 0
with open(filename, 'rb') as f:
for line_no, line in enumerate(f, start=1):
if pattern.search(line):
retv = 1
output.write(f'{filename}:{line_no}:')
output.write_line_b(line.rstrip(b'\r\n'))
return retv
def _process_filename_at_once(pattern: Pattern[bytes], filename: str) -> int:
retv = 0
with open(filename, 'rb') as f:
contents = f.read()
match = pattern.search(contents)
if match:
retv = 1
line_no = contents[:match.start()].count(b'\n')
output.write(f'{filename}:{line_no + 1}:')
matched_lines = match[0].split(b'\n')
matched_lines[0] = contents.split(b'\n')[line_no]
output.write_line_b(b'\n'.join(matched_lines))
return retv
def _process_filename_by_line_negated(
pattern: Pattern[bytes],
filename: str,
) -> int:
with open(filename, 'rb') as f:
for line in f:
if pattern.search(line):
return 0
else:
output.write_line(filename)
return 1
def _process_filename_at_once_negated(
pattern: Pattern[bytes],
filename: str,
) -> int:
with open(filename, 'rb') as f:
contents = f.read()
match = pattern.search(contents)
if match:
return 0
else:
output.write_line(filename)
return 1
class Choice(NamedTuple):
multiline: bool
negate: bool
FNS = {
Choice(multiline=True, negate=True): _process_filename_at_once_negated,
Choice(multiline=True, negate=False): _process_filename_at_once,
Choice(multiline=False, negate=True): _process_filename_by_line_negated,
Choice(multiline=False, negate=False): _process_filename_by_line,
}
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
exe = (sys.executable, '-m', __name__) + tuple(hook.args) + (hook.entry,)
return xargs(exe, file_args, color=color)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(
description=(
'grep-like finder using python regexes. Unlike grep, this tool '
'returns nonzero when it finds a match and zero otherwise. The '
'idea here being that matches are "problems".'
),
)
parser.add_argument('-i', '--ignore-case', action='store_true')
parser.add_argument('--multiline', action='store_true')
parser.add_argument('--negate', action='store_true')
parser.add_argument('pattern', help='python regex pattern.')
parser.add_argument('filenames', nargs='*')
args = parser.parse_args(argv)
flags = re.IGNORECASE if args.ignore_case else 0
if args.multiline:
flags |= re.MULTILINE | re.DOTALL
pattern = re.compile(args.pattern.encode(), flags)
retv = 0
process_fn = FNS[Choice(multiline=args.multiline, negate=args.negate)]
for filename in args.filenames:
retv |= process_fn(pattern, filename)
return retv
if __name__ == '__main__':
raise SystemExit(main())

View file

@ -0,0 +1,214 @@
import contextlib
import functools
import os
import sys
from typing import Dict
from typing import Generator
from typing import Optional
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import UNSET
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.parse_shebang import find_executable
from pre_commit.prefix import Prefix
from pre_commit.util import CalledProcessError
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output
from pre_commit.util import cmd_output_b
from pre_commit.util import win_exe
ENVIRONMENT_DIR = 'py_env'
@functools.lru_cache(maxsize=None)
def _version_info(exe: str) -> str:
prog = 'import sys;print(".".join(str(p) for p in sys.version_info))'
try:
return cmd_output(exe, '-S', '-c', prog)[1].strip()
except CalledProcessError:
return f'<<error retrieving version from {exe}>>'
def _read_pyvenv_cfg(filename: str) -> Dict[str, str]:
ret = {}
with open(filename, encoding='UTF-8') as f:
for line in f:
try:
k, v = line.split('=')
except ValueError: # blank line / comment / etc.
continue
else:
ret[k.strip()] = v.strip()
return ret
def bin_dir(venv: str) -> str:
"""On windows there's a different directory for the virtualenv"""
bin_part = 'Scripts' if os.name == 'nt' else 'bin'
return os.path.join(venv, bin_part)
def get_env_patch(venv: str) -> PatchesT:
return (
('PIP_DISABLE_PIP_VERSION_CHECK', '1'),
('PYTHONHOME', UNSET),
('VIRTUAL_ENV', venv),
('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
)
def _find_by_py_launcher(
version: str,
) -> Optional[str]: # pragma: no cover (windows only)
if version.startswith('python'):
num = version[len('python'):]
cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)')
env = dict(os.environ, PYTHONIOENCODING='UTF-8')
try:
return cmd_output(*cmd, env=env)[1].strip()
except CalledProcessError:
pass
return None
def _find_by_sys_executable() -> Optional[str]:
def _norm(path: str) -> Optional[str]:
_, exe = os.path.split(path.lower())
exe, _, _ = exe.partition('.exe')
if exe not in {'python', 'pythonw'} and find_executable(exe):
return exe
return None
# On linux, I see these common sys.executables:
#
# system `python`: /usr/bin/python -> python2.7
# system `python2`: /usr/bin/python2 -> python2.7
# virtualenv v: v/bin/python (will not return from this loop)
# virtualenv v -ppython2: v/bin/python -> python2
# virtualenv v -ppython2.7: v/bin/python -> python2.7
# virtualenv v -ppypy: v/bin/python -> v/bin/pypy
for path in (sys.executable, os.path.realpath(sys.executable)):
exe = _norm(path)
if exe:
return exe
return None
@functools.lru_cache(maxsize=1)
def get_default_version() -> str: # pragma: no cover (platform dependent)
# First attempt from `sys.executable` (or the realpath)
exe = _find_by_sys_executable()
if exe:
return exe
# Next try the `pythonX.X` executable
exe = f'python{sys.version_info[0]}.{sys.version_info[1]}'
if find_executable(exe):
return exe
if _find_by_py_launcher(exe):
return exe
# We tried!
return C.DEFAULT
def _sys_executable_matches(version: str) -> bool:
if version == 'python':
return True
elif not version.startswith('python'):
return False
try:
info = tuple(int(p) for p in version[len('python'):].split('.'))
except ValueError:
return False
return sys.version_info[:len(info)] == info
def norm_version(version: str) -> Optional[str]:
if version == C.DEFAULT: # use virtualenv's default
return None
elif _sys_executable_matches(version): # virtualenv defaults to our exe
return None
if os.name == 'nt': # pragma: no cover (windows)
version_exec = _find_by_py_launcher(version)
if version_exec:
return version_exec
# Try looking up by name
version_exec = find_executable(version)
if version_exec and version_exec != version:
return version_exec
# Otherwise assume it is a path
return os.path.expanduser(version)
@contextlib.contextmanager
def in_env(
prefix: Prefix,
language_version: str,
) -> Generator[None, None, None]:
directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
envdir = prefix.path(directory)
with envcontext(get_env_patch(envdir)):
yield
def healthy(prefix: Prefix, language_version: str) -> bool:
directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
envdir = prefix.path(directory)
pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg')
# created with "old" virtualenv
if not os.path.exists(pyvenv_cfg):
return False
exe_name = win_exe('python')
py_exe = prefix.path(bin_dir(envdir), exe_name)
cfg = _read_pyvenv_cfg(pyvenv_cfg)
return (
'version_info' in cfg and
# always use uncached lookup here in case we replaced an unhealthy env
_version_info.__wrapped__(py_exe) == cfg['version_info'] and (
'base-executable' not in cfg or
_version_info(cfg['base-executable']) == cfg['version_info']
)
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
venv_cmd = [sys.executable, '-mvirtualenv', envdir]
python = norm_version(version)
if python is not None:
venv_cmd.extend(('-p', python))
install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies)
with clean_path_on_failure(envdir):
cmd_output_b(*venv_cmd, cwd='/')
with in_env(prefix, version):
helpers.run_setup_cmd(prefix, install_cmd)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix, hook.language_version):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,155 @@
import contextlib
import os
import shlex
import shutil
from typing import Generator
from typing import Sequence
from typing import Tuple
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import UNSET
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output_b
ENVIRONMENT_DIR = 'renv'
RSCRIPT_OPTS = ('--no-save', '--no-restore', '--no-site-file', '--no-environ')
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def get_env_patch(venv: str) -> PatchesT:
return (
('R_PROFILE_USER', os.path.join(venv, 'activate.R')),
('RENV_PROJECT', UNSET),
)
@contextlib.contextmanager
def in_env(
prefix: Prefix,
language_version: str,
) -> Generator[None, None, None]:
envdir = _get_env_dir(prefix, language_version)
with envcontext(get_env_patch(envdir)):
yield
def _get_env_dir(prefix: Prefix, version: str) -> str:
return prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
def _prefix_if_non_local_file_entry(
entry: Sequence[str],
prefix: Prefix,
src: str,
) -> Sequence[str]:
if entry[1] == '-e':
return entry[1:]
else:
if src == 'local':
path = entry[1]
else:
path = prefix.path(entry[1])
return (path,)
def _rscript_exec() -> str:
return os.path.join(os.getenv('R_HOME', ''), 'Rscript')
def _entry_validate(entry: Sequence[str]) -> None:
"""
Allowed entries:
# Rscript -e expr
# Rscript path/to/file
"""
if entry[0] != 'Rscript':
raise ValueError('entry must start with `Rscript`.')
if entry[1] == '-e':
if len(entry) > 3:
raise ValueError('You can supply at most one expression.')
elif len(entry) > 2:
raise ValueError(
'The only valid syntax is `Rscript -e {expr}`',
'or `Rscript path/to/hook/script`',
)
def _cmd_from_hook(hook: Hook) -> Tuple[str, ...]:
entry = shlex.split(hook.entry)
_entry_validate(entry)
return (
*entry[:1], *RSCRIPT_OPTS,
*_prefix_if_non_local_file_entry(entry, hook.prefix, hook.src),
*hook.args,
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
env_dir = _get_env_dir(prefix, version)
with clean_path_on_failure(env_dir):
os.makedirs(env_dir, exist_ok=True)
shutil.copy(prefix.path('renv.lock'), env_dir)
shutil.copytree(prefix.path('renv'), os.path.join(env_dir, 'renv'))
cmd_output_b(
_rscript_exec(), '--vanilla', '-e',
f"""\
prefix_dir <- {prefix.prefix_dir!r}
options(
repos = c(CRAN = "https://cran.rstudio.com"),
renv.consent = TRUE
)
source("renv/activate.R")
renv::restore()
activate_statement <- paste0(
'suppressWarnings({{',
'old <- setwd("', getwd(), '"); ',
'source("renv/activate.R"); ',
'setwd(old); ',
'renv::load("', getwd(), '");}})'
)
writeLines(activate_statement, 'activate.R')
is_package <- tryCatch(
{{
path_desc <- file.path(prefix_dir, 'DESCRIPTION')
suppressWarnings(desc <- read.dcf(path_desc))
"Package" %in% colnames(desc)
}},
error = function(...) FALSE
)
if (is_package) {{
renv::install(prefix_dir)
}}
""",
cwd=env_dir,
)
if additional_dependencies:
with in_env(prefix, version):
cmd_output_b(
_rscript_exec(), *RSCRIPT_OPTS, '-e',
'renv::install(commandArgs(trailingOnly = TRUE))',
*additional_dependencies,
cwd=env_dir,
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix, hook.language_version):
return helpers.run_xargs(
hook, _cmd_from_hook(hook), file_args, color=color,
)

View file

@ -0,0 +1,151 @@
import contextlib
import functools
import os.path
import shutil
import tarfile
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import UNSET
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import CalledProcessError
from pre_commit.util import clean_path_on_failure
from pre_commit.util import resource_bytesio
ENVIRONMENT_DIR = 'rbenv'
healthy = helpers.basic_healthy
@functools.lru_cache(maxsize=1)
def get_default_version() -> str:
if all(helpers.exe_exists(exe) for exe in ('ruby', 'gem')):
return 'system'
else:
return C.DEFAULT
def get_env_patch(
venv: str,
language_version: str,
) -> PatchesT:
patches: PatchesT = (
('GEM_HOME', os.path.join(venv, 'gems')),
('GEM_PATH', UNSET),
('BUNDLE_IGNORE_CONFIG', '1'),
)
if language_version == 'system':
patches += (
(
'PATH', (
os.path.join(venv, 'gems', 'bin'), os.pathsep,
Var('PATH'),
),
),
)
else: # pragma: win32 no cover
patches += (
('RBENV_ROOT', venv),
(
'PATH', (
os.path.join(venv, 'gems', 'bin'), os.pathsep,
os.path.join(venv, 'shims'), os.pathsep,
os.path.join(venv, 'bin'), os.pathsep, Var('PATH'),
),
),
)
if language_version not in {'system', 'default'}: # pragma: win32 no cover
patches += (('RBENV_VERSION', language_version),)
return patches
@contextlib.contextmanager
def in_env(
prefix: Prefix,
language_version: str,
) -> Generator[None, None, None]:
envdir = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, language_version),
)
with envcontext(get_env_patch(envdir, language_version)):
yield
def _extract_resource(filename: str, dest: str) -> None:
with resource_bytesio(filename) as bio:
with tarfile.open(fileobj=bio) as tf:
tf.extractall(dest)
def _install_rbenv(
prefix: Prefix,
version: str,
) -> None: # pragma: win32 no cover
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
_extract_resource('rbenv.tar.gz', prefix.path('.'))
shutil.move(prefix.path('rbenv'), prefix.path(directory))
# Only install ruby-build if the version is specified
if version != C.DEFAULT:
plugins_dir = prefix.path(directory, 'plugins')
_extract_resource('ruby-download.tar.gz', plugins_dir)
_extract_resource('ruby-build.tar.gz', plugins_dir)
def _install_ruby(
prefix: Prefix,
version: str,
) -> None: # pragma: win32 no cover
try:
helpers.run_setup_cmd(prefix, ('rbenv', 'download', version))
except CalledProcessError: # pragma: no cover (usually find with download)
# Failed to download from mirror for some reason, build it instead
helpers.run_setup_cmd(prefix, ('rbenv', 'install', version))
def install_environment(
prefix: Prefix, version: str, additional_dependencies: Sequence[str],
) -> None:
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(prefix.path(directory)):
if version != 'system': # pragma: win32 no cover
_install_rbenv(prefix, version)
with in_env(prefix, version):
# Need to call this before installing so rbenv's directories
# are set up
helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-'))
if version != C.DEFAULT:
_install_ruby(prefix, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(prefix, ('rbenv', 'rehash'))
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix, ('gem', 'build', *prefix.star('.gemspec')),
)
helpers.run_setup_cmd(
prefix,
(
'gem', 'install',
'--no-document', '--no-format-executable',
*prefix.star('.gem'), *additional_dependencies,
),
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix, hook.language_version):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,106 @@
import contextlib
import os.path
from typing import Generator
from typing import Sequence
from typing import Set
from typing import Tuple
import toml
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output_b
ENVIRONMENT_DIR = 'rustenv'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
def get_env_patch(target_dir: str) -> PatchesT:
return (
('PATH', (os.path.join(target_dir, 'bin'), os.pathsep, Var('PATH'))),
)
@contextlib.contextmanager
def in_env(prefix: Prefix) -> Generator[None, None, None]:
target_dir = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
with envcontext(get_env_patch(target_dir)):
yield
def _add_dependencies(
cargo_toml_path: str,
additional_dependencies: Set[str],
) -> None:
with open(cargo_toml_path, 'r+') as f:
cargo_toml = toml.load(f)
cargo_toml.setdefault('dependencies', {})
for dep in additional_dependencies:
name, _, spec = dep.partition(':')
cargo_toml['dependencies'][name] = spec or '*'
f.seek(0)
toml.dump(cargo_toml, f)
f.truncate()
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('rust', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# There are two cases where we might want to specify more dependencies:
# as dependencies for the library being built, and as binary packages
# to be `cargo install`'d.
#
# Unlike e.g. Python, if we just `cargo install` a library, it won't be
# used for compilation. And if we add a crate providing a binary to the
# `Cargo.toml`, the binary won't be built.
#
# Because of this, we allow specifying "cli" dependencies by prefixing
# with 'cli:'.
cli_deps = {
dep for dep in additional_dependencies if dep.startswith('cli:')
}
lib_deps = set(additional_dependencies) - cli_deps
if len(lib_deps) > 0:
_add_dependencies(prefix.path('Cargo.toml'), lib_deps)
with clean_path_on_failure(directory):
packages_to_install: Set[Tuple[str, ...]] = {('--path', '.')}
for cli_dep in cli_deps:
cli_dep = cli_dep[len('cli:'):]
package, _, version = cli_dep.partition(':')
if version != '':
packages_to_install.add((package, '--version', version))
else:
packages_to_install.add((package,))
for args in packages_to_install:
cmd_output_b(
'cargo', 'install', '--bins', '--root', directory, *args,
cwd=prefix.prefix_dir,
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,19 @@
from typing import Sequence
from typing import Tuple
from pre_commit.hook import Hook
from pre_commit.languages import helpers
ENVIRONMENT_DIR = None
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
install_environment = helpers.no_install
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
cmd = (hook.prefix.path(hook.cmd[0]), *hook.cmd[1:])
return helpers.run_xargs(hook, cmd, file_args, color=color)

View file

@ -0,0 +1,64 @@
import contextlib
import os
from typing import Generator
from typing import Sequence
from typing import Tuple
import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import PatchesT
from pre_commit.envcontext import Var
from pre_commit.hook import Hook
from pre_commit.languages import helpers
from pre_commit.prefix import Prefix
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output_b
ENVIRONMENT_DIR = 'swift_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
BUILD_DIR = '.build'
BUILD_CONFIG = 'release'
def get_env_patch(venv: str) -> PatchesT: # pragma: win32 no cover
bin_path = os.path.join(venv, BUILD_DIR, BUILD_CONFIG)
return (('PATH', (bin_path, os.pathsep, Var('PATH'))),)
@contextlib.contextmanager # pragma: win32 no cover
def in_env(prefix: Prefix) -> Generator[None, None, None]:
envdir = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
with envcontext(get_env_patch(envdir)):
yield
def install_environment(
prefix: Prefix, version: str, additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('swift', version)
helpers.assert_no_additional_deps('swift', additional_dependencies)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
cmd_output_b(
'swift', 'build',
'-C', prefix.prefix_dir,
'-c', BUILD_CONFIG,
'--build-path', os.path.join(directory, BUILD_DIR),
)
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]: # pragma: win32 no cover
with in_env(hook.prefix):
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,19 @@
from typing import Sequence
from typing import Tuple
from pre_commit.hook import Hook
from pre_commit.languages import helpers
ENVIRONMENT_DIR = None
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy
install_environment = helpers.no_install
def run_hook(
hook: Hook,
file_args: Sequence[str],
color: bool,
) -> Tuple[int, bytes]:
return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

View file

@ -0,0 +1,40 @@
import contextlib
import logging
from typing import Generator
from pre_commit import color
from pre_commit import output
logger = logging.getLogger('pre_commit')
LOG_LEVEL_COLORS = {
'DEBUG': '',
'INFO': '',
'WARNING': color.YELLOW,
'ERROR': color.RED,
}
class LoggingHandler(logging.Handler):
def __init__(self, use_color: bool) -> None:
super().__init__()
self.use_color = use_color
def emit(self, record: logging.LogRecord) -> None:
level_msg = color.format_color(
f'[{record.levelname}]',
LOG_LEVEL_COLORS[record.levelname],
self.use_color,
)
output.write_line(f'{level_msg} {record.getMessage()}')
@contextlib.contextmanager
def logging_handler(use_color: bool) -> Generator[None, None, None]:
handler = LoggingHandler(use_color)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
try:
yield
finally:
logger.removeHandler(handler)

View file

@ -0,0 +1,414 @@
import argparse
import logging
import os
import sys
from typing import Any
from typing import Optional
from typing import Sequence
from typing import Union
import pre_commit.constants as C
from pre_commit import git
from pre_commit.color import add_color_option
from pre_commit.commands.autoupdate import autoupdate
from pre_commit.commands.clean import clean
from pre_commit.commands.gc import gc
from pre_commit.commands.hook_impl import hook_impl
from pre_commit.commands.init_templatedir import init_templatedir
from pre_commit.commands.install_uninstall import install
from pre_commit.commands.install_uninstall import install_hooks
from pre_commit.commands.install_uninstall import uninstall
from pre_commit.commands.migrate_config import migrate_config
from pre_commit.commands.run import run
from pre_commit.commands.sample_config import sample_config
from pre_commit.commands.try_repo import try_repo
from pre_commit.error_handler import error_handler
from pre_commit.logging_handler import logging_handler
from pre_commit.store import Store
logger = logging.getLogger('pre_commit')
# https://github.com/pre-commit/pre-commit/issues/217
# On OSX, making a virtualenv using pyvenv at . causes `virtualenv` and `pip`
# to install packages to the wrong place. We don't want anything to deal with
# pyvenv
os.environ.pop('__PYVENV_LAUNCHER__', None)
COMMANDS_NO_GIT = {'clean', 'gc', 'init-templatedir', 'sample-config'}
def _add_config_option(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-c', '--config', default=C.CONFIG_FILE,
help='Path to alternate config file',
)
class AppendReplaceDefault(argparse.Action):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.appended = False
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Union[str, Sequence[str], None],
option_string: Optional[str] = None,
) -> None:
if not self.appended:
setattr(namespace, self.dest, [])
self.appended = True
getattr(namespace, self.dest).append(values)
def _add_hook_type_option(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
'-t', '--hook-type', choices=(
'pre-commit', 'pre-merge-commit', 'pre-push', 'prepare-commit-msg',
'commit-msg', 'post-commit', 'post-checkout', 'post-merge',
'post-rewrite',
),
action=AppendReplaceDefault,
default=['pre-commit'],
dest='hook_types',
)
def _add_run_options(parser: argparse.ArgumentParser) -> None:
parser.add_argument('hook', nargs='?', help='A single hook-id to run')
parser.add_argument('--verbose', '-v', action='store_true', default=False)
mutex_group = parser.add_mutually_exclusive_group(required=False)
mutex_group.add_argument(
'--all-files', '-a', action='store_true', default=False,
help='Run on all the files in the repo.',
)
mutex_group.add_argument(
'--files', nargs='*', default=[],
help='Specific filenames to run hooks on.',
)
parser.add_argument(
'--show-diff-on-failure', action='store_true',
help='When hooks fail, run `git diff` directly afterward.',
)
parser.add_argument(
'--hook-stage', choices=C.STAGES, default='commit',
help='The stage during which the hook is fired. One of %(choices)s',
)
parser.add_argument(
'--remote-branch', help='Remote branch ref used by `git push`.',
)
parser.add_argument(
'--local-branch', help='Local branch ref used by `git push`.',
)
parser.add_argument(
'--from-ref', '--source', '-s',
help=(
'(for usage with `--from-ref`) -- this option represents the '
'original ref in a `from_ref...to_ref` diff expression. '
'For `pre-push` hooks, this represents the branch you are pushing '
'to. '
'For `post-checkout` hooks, this represents the branch that was '
'previously checked out.'
),
)
parser.add_argument(
'--to-ref', '--origin', '-o',
help=(
'(for usage with `--to-ref`) -- this option represents the '
'destination ref in a `from_ref...to_ref` diff expression. '
'For `pre-push` hooks, this represents the branch being pushed. '
'For `post-checkout` hooks, this represents the branch that is '
'now checked out.'
),
)
parser.add_argument(
'--commit-msg-filename',
help='Filename to check when running during `commit-msg`',
)
parser.add_argument(
'--remote-name', help='Remote name used by `git push`.',
)
parser.add_argument('--remote-url', help='Remote url used by `git push`.')
parser.add_argument(
'--checkout-type',
help=(
'Indicates whether the checkout was a branch checkout '
'(changing branches, flag=1) or a file checkout (retrieving a '
'file from the index, flag=0).'
),
)
parser.add_argument(
'--is-squash-merge',
help=(
'During a post-merge hook, indicates whether the merge was a '
'squash merge'
),
)
parser.add_argument(
'--rewrite-command',
help=(
'During a post-rewrite hook, specifies the command that invoked '
'the rewrite'
),
)
def _adjust_args_and_chdir(args: argparse.Namespace) -> None:
# `--config` was specified relative to the non-root working directory
if os.path.exists(args.config):
args.config = os.path.abspath(args.config)
if args.command in {'run', 'try-repo'}:
args.files = [os.path.abspath(filename) for filename in args.files]
if args.command == 'try-repo' and os.path.exists(args.repo):
args.repo = os.path.abspath(args.repo)
toplevel = git.get_root()
os.chdir(toplevel)
args.config = os.path.relpath(args.config)
if args.command in {'run', 'try-repo'}:
args.files = [os.path.relpath(filename) for filename in args.files]
if args.command == 'try-repo' and os.path.exists(args.repo):
args.repo = os.path.relpath(args.repo)
def main(argv: Optional[Sequence[str]] = None) -> int:
argv = argv if argv is not None else sys.argv[1:]
parser = argparse.ArgumentParser(prog='pre-commit')
# https://stackoverflow.com/a/8521644/812183
parser.add_argument(
'-V', '--version',
action='version',
version=f'%(prog)s {C.VERSION}',
)
subparsers = parser.add_subparsers(dest='command')
autoupdate_parser = subparsers.add_parser(
'autoupdate',
help="Auto-update pre-commit config to the latest repos' versions.",
)
add_color_option(autoupdate_parser)
_add_config_option(autoupdate_parser)
autoupdate_parser.add_argument(
'--bleeding-edge', action='store_true',
help=(
'Update to the bleeding edge of `master` instead of the latest '
'tagged version (the default behavior).'
),
)
autoupdate_parser.add_argument(
'--freeze', action='store_true',
help='Store "frozen" hashes in `rev` instead of tag names',
)
autoupdate_parser.add_argument(
'--repo', dest='repos', action='append', metavar='REPO',
help='Only update this repository -- may be specified multiple times.',
)
clean_parser = subparsers.add_parser(
'clean', help='Clean out pre-commit files.',
)
add_color_option(clean_parser)
_add_config_option(clean_parser)
hook_impl_parser = subparsers.add_parser('hook-impl')
add_color_option(hook_impl_parser)
_add_config_option(hook_impl_parser)
hook_impl_parser.add_argument('--hook-type')
hook_impl_parser.add_argument('--hook-dir')
hook_impl_parser.add_argument(
'--skip-on-missing-config', action='store_true',
)
hook_impl_parser.add_argument(dest='rest', nargs=argparse.REMAINDER)
gc_parser = subparsers.add_parser('gc', help='Clean unused cached repos.')
add_color_option(gc_parser)
_add_config_option(gc_parser)
init_templatedir_parser = subparsers.add_parser(
'init-templatedir',
help=(
'Install hook script in a directory intended for use with '
'`git config init.templateDir`.'
),
)
add_color_option(init_templatedir_parser)
_add_config_option(init_templatedir_parser)
init_templatedir_parser.add_argument(
'directory', help='The directory in which to write the hook script.',
)
init_templatedir_parser.add_argument(
'--no-allow-missing-config',
action='store_false',
dest='allow_missing_config',
help='Assume cloned repos should have a `pre-commit` config.',
)
_add_hook_type_option(init_templatedir_parser)
install_parser = subparsers.add_parser(
'install', help='Install the pre-commit script.',
)
add_color_option(install_parser)
_add_config_option(install_parser)
install_parser.add_argument(
'-f', '--overwrite', action='store_true',
help='Overwrite existing hooks / remove migration mode.',
)
install_parser.add_argument(
'--install-hooks', action='store_true',
help=(
'Whether to install hook environments for all environments '
'in the config file.'
),
)
_add_hook_type_option(install_parser)
install_parser.add_argument(
'--allow-missing-config', action='store_true', default=False,
help=(
'Whether to allow a missing `pre-commit` configuration file '
'or exit with a failure code.'
),
)
install_hooks_parser = subparsers.add_parser(
'install-hooks',
help=(
'Install hook environments for all environments in the config '
'file. You may find `pre-commit install --install-hooks` more '
'useful.'
),
)
add_color_option(install_hooks_parser)
_add_config_option(install_hooks_parser)
migrate_config_parser = subparsers.add_parser(
'migrate-config',
help='Migrate list configuration to new map configuration.',
)
add_color_option(migrate_config_parser)
_add_config_option(migrate_config_parser)
run_parser = subparsers.add_parser('run', help='Run hooks.')
add_color_option(run_parser)
_add_config_option(run_parser)
_add_run_options(run_parser)
sample_config_parser = subparsers.add_parser(
'sample-config', help=f'Produce a sample {C.CONFIG_FILE} file',
)
add_color_option(sample_config_parser)
_add_config_option(sample_config_parser)
try_repo_parser = subparsers.add_parser(
'try-repo',
help='Try the hooks in a repository, useful for developing new hooks.',
)
add_color_option(try_repo_parser)
_add_config_option(try_repo_parser)
try_repo_parser.add_argument(
'repo', help='Repository to source hooks from.',
)
try_repo_parser.add_argument(
'--ref', '--rev',
help=(
'Manually select a rev to run against, otherwise the `HEAD` '
'revision will be used.'
),
)
_add_run_options(try_repo_parser)
uninstall_parser = subparsers.add_parser(
'uninstall', help='Uninstall the pre-commit script.',
)
add_color_option(uninstall_parser)
_add_config_option(uninstall_parser)
_add_hook_type_option(uninstall_parser)
help = subparsers.add_parser(
'help', help='Show help for a specific command.',
)
help.add_argument('help_cmd', nargs='?', help='Command to show help for.')
# argparse doesn't really provide a way to use a `default` subparser
if len(argv) == 0:
argv = ['run']
args = parser.parse_args(argv)
if args.command == 'help' and args.help_cmd:
parser.parse_args([args.help_cmd, '--help'])
elif args.command == 'help':
parser.parse_args(['--help'])
with error_handler(), logging_handler(args.color):
git.check_for_cygwin_mismatch()
if args.command not in COMMANDS_NO_GIT:
_adjust_args_and_chdir(args)
store = Store()
store.mark_config_used(args.config)
if args.command == 'autoupdate':
return autoupdate(
args.config, store,
tags_only=not args.bleeding_edge,
freeze=args.freeze,
repos=args.repos,
)
elif args.command == 'clean':
return clean(store)
elif args.command == 'gc':
return gc(store)
elif args.command == 'hook-impl':
return hook_impl(
store,
config=args.config,
color=args.color,
hook_type=args.hook_type,
hook_dir=args.hook_dir,
skip_on_missing_config=args.skip_on_missing_config,
args=args.rest[1:],
)
elif args.command == 'install':
return install(
args.config, store,
hook_types=args.hook_types,
overwrite=args.overwrite,
hooks=args.install_hooks,
skip_on_missing_config=args.allow_missing_config,
)
elif args.command == 'init-templatedir':
return init_templatedir(
args.config, store, args.directory,
hook_types=args.hook_types,
skip_on_missing_config=args.allow_missing_config,
)
elif args.command == 'install-hooks':
return install_hooks(args.config, store)
elif args.command == 'migrate-config':
return migrate_config(args.config)
elif args.command == 'run':
return run(args.config, store, args)
elif args.command == 'sample-config':
return sample_config()
elif args.command == 'try-repo':
return try_repo(args)
elif args.command == 'uninstall':
return uninstall(hook_types=args.hook_types)
else:
raise NotImplementedError(
f'Command {args.command} not implemented.',
)
raise AssertionError(
f'Command {args.command} failed to exit with a returncode',
)
if __name__ == '__main__':
raise SystemExit(main())

Some files were not shown because too many files have changed in this diff Show more