This commit is contained in:
Waylon Walker 2022-03-31 20:20:07 -05:00
commit 38355d2442
No known key found for this signature in database
GPG key ID: 66E2BF2B4190EFE4
9083 changed files with 1225834 additions and 0 deletions

View file

@ -0,0 +1,74 @@
"""Top-level module for Flake8.
This module
- initializes logging for the command-line tool
- tracks the version of the package
- provides a way to configure logging for the command-line tool
.. autofunction:: flake8.configure_logging
"""
import logging
import sys
from typing import Type
LOG = logging.getLogger(__name__)
LOG.addHandler(logging.NullHandler())
__version__ = "4.0.1"
__version_info__ = tuple(int(i) for i in __version__.split(".") if i.isdigit())
# There is nothing lower than logging.DEBUG (10) in the logging library,
# but we want an extra level to avoid being too verbose when using -vv.
_EXTRA_VERBOSE = 5
logging.addLevelName(_EXTRA_VERBOSE, "VERBOSE")
_VERBOSITY_TO_LOG_LEVEL = {
# output more than warnings but not debugging info
1: logging.INFO, # INFO is a numerical level of 20
# output debugging information
2: logging.DEBUG, # DEBUG is a numerical level of 10
# output extra verbose debugging information
3: _EXTRA_VERBOSE,
}
LOG_FORMAT = (
"%(name)-25s %(processName)-11s %(relativeCreated)6d "
"%(levelname)-8s %(message)s"
)
def configure_logging(verbosity, filename=None, logformat=LOG_FORMAT):
"""Configure logging for flake8.
:param int verbosity:
How verbose to be in logging information.
:param str filename:
Name of the file to append log information to.
If ``None`` this will log to ``sys.stderr``.
If the name is "stdout" or "stderr" this will log to the appropriate
stream.
"""
if verbosity <= 0:
return
if verbosity > 3:
verbosity = 3
log_level = _VERBOSITY_TO_LOG_LEVEL[verbosity]
if not filename or filename in ("stderr", "stdout"):
fileobj = getattr(sys, filename or "stderr")
handler_cls: Type[logging.Handler] = logging.StreamHandler
else:
fileobj = filename
handler_cls = logging.FileHandler
handler = handler_cls(fileobj)
handler.setFormatter(logging.Formatter(logformat))
LOG.addHandler(handler)
LOG.setLevel(log_level)
LOG.debug(
"Added a %s logging handler to logger root at %s", filename, __name__
)

View file

@ -0,0 +1,4 @@
"""Module allowing for ``python -m flake8 ...``."""
from flake8.main import cli
cli.main()

View file

@ -0,0 +1,9 @@
"""Expose backports in a single place."""
import sys
if sys.version_info >= (3, 8): # pragma: no cover (PY38+)
import importlib.metadata as importlib_metadata
else: # pragma: no cover (<PY38)
import importlib_metadata
__all__ = ("importlib_metadata",)

View file

@ -0,0 +1,5 @@
"""Module containing all public entry-points for Flake8.
This is the only submodule in Flake8 with a guaranteed stable API. All other
submodules are considered internal only and are subject to change.
"""

View file

@ -0,0 +1,215 @@
"""Module containing shims around Flake8 2.x behaviour.
Previously, users would import :func:`get_style_guide` from ``flake8.engine``.
In 3.0 we no longer have an "engine" module but we maintain the API from it.
"""
import argparse
import logging
import os.path
import flake8
from flake8.formatting import base as formatter
from flake8.main import application as app
from flake8.options import config
LOG = logging.getLogger(__name__)
__all__ = ("get_style_guide",)
def get_style_guide(**kwargs):
r"""Provision a StyleGuide for use.
:param \*\*kwargs:
Keyword arguments that provide some options for the StyleGuide.
:returns:
An initialized StyleGuide
:rtype:
:class:`StyleGuide`
"""
application = app.Application()
prelim_opts, remaining_args = application.parse_preliminary_options([])
flake8.configure_logging(prelim_opts.verbose, prelim_opts.output_file)
config_finder = config.ConfigFileFinder(
application.program,
prelim_opts.append_config,
config_file=prelim_opts.config,
ignore_config_files=prelim_opts.isolated,
)
application.find_plugins(config_finder)
application.register_plugin_options()
application.parse_configuration_and_cli(
config_finder,
remaining_args,
)
# We basically want application.initialize to be called but with these
# options set instead before we make our formatter, notifier, internal
# style guide and file checker manager.
options = application.options
for key, value in kwargs.items():
try:
getattr(options, key)
setattr(options, key, value)
except AttributeError:
LOG.error('Could not update option "%s"', key)
application.make_formatter()
application.make_guide()
application.make_file_checker_manager()
return StyleGuide(application)
class StyleGuide:
"""Public facing object that mimic's Flake8 2.0's StyleGuide.
.. note::
There are important changes in how this object behaves compared to
the StyleGuide object provided in Flake8 2.x.
.. warning::
This object should not be instantiated directly by users.
.. versionchanged:: 3.0.0
"""
def __init__(self, application):
"""Initialize our StyleGuide."""
self._application = application
self._file_checker_manager = application.file_checker_manager
@property
def options(self) -> argparse.Namespace:
"""Return application's options.
An instance of :class:`argparse.Namespace` containing parsed options.
"""
return self._application.options
@property
def paths(self):
"""Return the extra arguments passed as paths."""
return self._application.paths
def check_files(self, paths=None):
"""Run collected checks on the files provided.
This will check the files passed in and return a :class:`Report`
instance.
:param list paths:
List of filenames (or paths) to check.
:returns:
Object that mimic's Flake8 2.0's Reporter class.
:rtype:
flake8.api.legacy.Report
"""
self._application.run_checks(paths)
self._application.report_errors()
return Report(self._application)
def excluded(self, filename, parent=None):
"""Determine if a file is excluded.
:param str filename:
Path to the file to check if it is excluded.
:param str parent:
Name of the parent directory containing the file.
:returns:
True if the filename is excluded, False otherwise.
:rtype:
bool
"""
return self._file_checker_manager.is_path_excluded(filename) or (
parent
and self._file_checker_manager.is_path_excluded(
os.path.join(parent, filename)
)
)
def init_report(self, reporter=None):
"""Set up a formatter for this run of Flake8."""
if reporter is None:
return
if not issubclass(reporter, formatter.BaseFormatter):
raise ValueError(
"Report should be subclass of "
"flake8.formatter.BaseFormatter."
)
self._application.formatter = None
self._application.make_formatter(reporter)
self._application.guide = None
# NOTE(sigmavirus24): This isn't the intended use of
# Application#make_guide but it works pretty well.
# Stop cringing... I know it's gross.
self._application.make_guide()
self._application.file_checker_manager = None
self._application.make_file_checker_manager()
def input_file(self, filename, lines=None, expected=None, line_offset=0):
"""Run collected checks on a single file.
This will check the file passed in and return a :class:`Report`
instance.
:param str filename:
The path to the file to check.
:param list lines:
Ignored since Flake8 3.0.
:param expected:
Ignored since Flake8 3.0.
:param int line_offset:
Ignored since Flake8 3.0.
:returns:
Object that mimic's Flake8 2.0's Reporter class.
:rtype:
flake8.api.legacy.Report
"""
return self.check_files([filename])
class Report:
"""Public facing object that mimic's Flake8 2.0's API.
.. note::
There are important changes in how this object behaves compared to
the object provided in Flake8 2.x.
.. warning::
This should not be instantiated by users.
.. versionchanged:: 3.0.0
"""
def __init__(self, application):
"""Initialize the Report for the user.
.. warning:: This should not be instantiated by users.
"""
self._application = application
self._style_guide = application.guide
self._stats = self._style_guide.stats
@property
def total_errors(self):
"""Return the total number of errors."""
return self._application.result_count
def get_statistics(self, violation):
"""Get the list of occurrences of a violation.
:returns:
List of occurrences of a violation formatted as:
{Count} {Error Code} {Message}, e.g.,
``8 E531 Some error message about the error``
:rtype:
list
"""
return [
f"{s.count} {s.error_code} {s.message}"
for s in self._stats.statistics_for(violation)
]

View file

@ -0,0 +1,705 @@
"""Checker Manager and Checker classes."""
import collections
import errno
import itertools
import logging
import signal
import tokenize
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from flake8 import defaults
from flake8 import exceptions
from flake8 import processor
from flake8 import utils
try:
import multiprocessing.pool
except ImportError:
multiprocessing = None # type: ignore
Results = List[Tuple[str, int, int, str, Optional[str]]]
LOG = logging.getLogger(__name__)
SERIAL_RETRY_ERRNOS = {
# ENOSPC: Added by sigmavirus24
# > On some operating systems (OSX), multiprocessing may cause an
# > ENOSPC error while trying to trying to create a Semaphore.
# > In those cases, we should replace the customized Queue Report
# > class with pep8's StandardReport class to ensure users don't run
# > into this problem.
# > (See also: https://github.com/pycqa/flake8/issues/117)
errno.ENOSPC,
# NOTE(sigmavirus24): When adding to this list, include the reasoning
# on the lines before the error code and always append your error
# code. Further, please always add a trailing `,` to reduce the visual
# noise in diffs.
}
def _multiprocessing_is_fork(): # type () -> bool
"""Class state is only preserved when using the `fork` strategy."""
return multiprocessing and multiprocessing.get_start_method() == "fork"
class Manager:
"""Manage the parallelism and checker instances for each plugin and file.
This class will be responsible for the following:
- Determining the parallelism of Flake8, e.g.:
* Do we use :mod:`multiprocessing` or is it unavailable?
* Do we automatically decide on the number of jobs to use or did the
user provide that?
- Falling back to a serial way of processing files if we run into an
OSError related to :mod:`multiprocessing`
- Organizing the results of each checker so we can group the output
together and make our output deterministic.
"""
def __init__(self, style_guide, arguments, checker_plugins):
"""Initialize our Manager instance.
:param style_guide:
The instantiated style guide for this instance of Flake8.
:type style_guide:
flake8.style_guide.StyleGuide
:param list arguments:
The extra arguments parsed from the CLI (if any)
:param checker_plugins:
The plugins representing checks parsed from entry-points.
:type checker_plugins:
flake8.plugins.manager.Checkers
"""
self.arguments = arguments
self.style_guide = style_guide
self.options = style_guide.options
self.checks = checker_plugins
self.jobs = self._job_count()
self._all_checkers: List[FileChecker] = []
self.checkers: List[FileChecker] = []
self.statistics = {
"files": 0,
"logical lines": 0,
"physical lines": 0,
"tokens": 0,
}
self.exclude = tuple(
itertools.chain(self.options.exclude, self.options.extend_exclude)
)
def _process_statistics(self):
for checker in self.checkers:
for statistic in defaults.STATISTIC_NAMES:
self.statistics[statistic] += checker.statistics[statistic]
self.statistics["files"] += len(self.checkers)
def _job_count(self) -> int:
# First we walk through all of our error cases:
# - multiprocessing library is not present
# - we're running on windows in which case we know we have significant
# implementation issues
# - the user provided stdin and that's not something we can handle
# well
# - we're processing a diff, which again does not work well with
# multiprocessing and which really shouldn't require multiprocessing
# - the user provided some awful input
if not _multiprocessing_is_fork():
LOG.warning(
"The multiprocessing module is not available. "
"Ignoring --jobs arguments."
)
return 0
if utils.is_using_stdin(self.arguments):
LOG.warning(
"The --jobs option is not compatible with supplying "
"input using - . Ignoring --jobs arguments."
)
return 0
if self.options.diff:
LOG.warning(
"The --diff option was specified with --jobs but "
"they are not compatible. Ignoring --jobs arguments."
)
return 0
jobs = self.options.jobs
# If the value is "auto", we want to let the multiprocessing library
# decide the number based on the number of CPUs. However, if that
# function is not implemented for this particular value of Python we
# default to 1
if jobs.is_auto:
try:
return multiprocessing.cpu_count()
except NotImplementedError:
return 0
# Otherwise, we know jobs should be an integer and we can just convert
# it to an integer
return jobs.n_jobs
def _handle_results(self, filename, results):
style_guide = self.style_guide
reported_results_count = 0
for (error_code, line_number, column, text, physical_line) in results:
reported_results_count += style_guide.handle_error(
code=error_code,
filename=filename,
line_number=line_number,
column_number=column,
text=text,
physical_line=physical_line,
)
return reported_results_count
def is_path_excluded(self, path: str) -> bool:
"""Check if a path is excluded.
:param str path:
Path to check against the exclude patterns.
:returns:
True if there are exclude patterns and the path matches,
otherwise False.
:rtype:
bool
"""
if path == "-":
if self.options.stdin_display_name == "stdin":
return False
path = self.options.stdin_display_name
return utils.matches_filename(
path,
patterns=self.exclude,
log_message='"%(path)s" has %(whether)sbeen excluded',
logger=LOG,
)
def make_checkers(self, paths: Optional[List[str]] = None) -> None:
"""Create checkers for each file."""
if paths is None:
paths = self.arguments
if not paths:
paths = ["."]
filename_patterns = self.options.filename
running_from_diff = self.options.diff
# NOTE(sigmavirus24): Yes this is a little unsightly, but it's our
# best solution right now.
def should_create_file_checker(filename, argument):
"""Determine if we should create a file checker."""
matches_filename_patterns = utils.fnmatch(
filename, filename_patterns
)
is_stdin = filename == "-"
# NOTE(sigmavirus24): If a user explicitly specifies something,
# e.g, ``flake8 bin/script`` then we should run Flake8 against
# that. Since should_create_file_checker looks to see if the
# filename patterns match the filename, we want to skip that in
# the event that the argument and the filename are identical.
# If it was specified explicitly, the user intended for it to be
# checked.
explicitly_provided = not running_from_diff and (
argument == filename
)
return (
explicitly_provided or matches_filename_patterns
) or is_stdin
checks = self.checks.to_dictionary()
self._all_checkers = [
FileChecker(filename, checks, self.options)
for argument in paths
for filename in utils.filenames_from(
argument, self.is_path_excluded
)
if should_create_file_checker(filename, argument)
]
self.checkers = [c for c in self._all_checkers if c.should_process]
LOG.info("Checking %d files", len(self.checkers))
def report(self) -> Tuple[int, int]:
"""Report all of the errors found in the managed file checkers.
This iterates over each of the checkers and reports the errors sorted
by line number.
:returns:
A tuple of the total results found and the results reported.
:rtype:
tuple(int, int)
"""
results_reported = results_found = 0
for checker in self._all_checkers:
results = sorted(checker.results, key=lambda tup: (tup[1], tup[2]))
filename = checker.display_name
with self.style_guide.processing_file(filename):
results_reported += self._handle_results(filename, results)
results_found += len(results)
return (results_found, results_reported)
def run_parallel(self) -> None:
"""Run the checkers in parallel."""
# fmt: off
final_results: Dict[str, List[Tuple[str, int, int, str, Optional[str]]]] = collections.defaultdict(list) # noqa: E501
final_statistics: Dict[str, Dict[str, int]] = collections.defaultdict(dict) # noqa: E501
# fmt: on
pool = _try_initialize_processpool(self.jobs)
if pool is None:
self.run_serial()
return
pool_closed = False
try:
pool_map = pool.imap_unordered(
_run_checks,
self.checkers,
chunksize=calculate_pool_chunksize(
len(self.checkers), self.jobs
),
)
for ret in pool_map:
filename, results, statistics = ret
final_results[filename] = results
final_statistics[filename] = statistics
pool.close()
pool.join()
pool_closed = True
finally:
if not pool_closed:
pool.terminate()
pool.join()
for checker in self.checkers:
filename = checker.display_name
checker.results = final_results[filename]
checker.statistics = final_statistics[filename]
def run_serial(self) -> None:
"""Run the checkers in serial."""
for checker in self.checkers:
checker.run_checks()
def run(self) -> None:
"""Run all the checkers.
This will intelligently decide whether to run the checks in parallel
or whether to run them in serial.
If running the checks in parallel causes a problem (e.g.,
https://github.com/pycqa/flake8/issues/117) this also implements
fallback to serial processing.
"""
try:
if self.jobs > 1 and len(self.checkers) > 1:
self.run_parallel()
else:
self.run_serial()
except KeyboardInterrupt:
LOG.warning("Flake8 was interrupted by the user")
raise exceptions.EarlyQuit("Early quit while running checks")
def start(self, paths=None):
"""Start checking files.
:param list paths:
Path names to check. This is passed directly to
:meth:`~Manager.make_checkers`.
"""
LOG.info("Making checkers")
self.make_checkers(paths)
def stop(self):
"""Stop checking files."""
self._process_statistics()
class FileChecker:
"""Manage running checks for a file and aggregate the results."""
def __init__(self, filename, checks, options):
"""Initialize our file checker.
:param str filename:
Name of the file to check.
:param checks:
The plugins registered to check the file.
:type checks:
dict
:param options:
Parsed option values from config and command-line.
:type options:
argparse.Namespace
"""
self.options = options
self.filename = filename
self.checks = checks
self.results: Results = []
self.statistics = {
"tokens": 0,
"logical lines": 0,
"physical lines": 0,
}
self.processor = self._make_processor()
self.display_name = filename
self.should_process = False
if self.processor is not None:
self.display_name = self.processor.filename
self.should_process = not self.processor.should_ignore_file()
self.statistics["physical lines"] = len(self.processor.lines)
def __repr__(self) -> str:
"""Provide helpful debugging representation."""
return f"FileChecker for {self.filename}"
def _make_processor(self) -> Optional[processor.FileProcessor]:
try:
return processor.FileProcessor(self.filename, self.options)
except OSError as e:
# If we can not read the file due to an IOError (e.g., the file
# does not exist or we do not have the permissions to open it)
# then we need to format that exception for the user.
# NOTE(sigmavirus24): Historically, pep8 has always reported this
# as an E902. We probably *want* a better error code for this
# going forward.
self.report("E902", 0, 0, f"{type(e).__name__}: {e}")
return None
def report(
self,
error_code: Optional[str],
line_number: int,
column: int,
text: str,
) -> str:
"""Report an error by storing it in the results list."""
if error_code is None:
error_code, text = text.split(" ", 1)
# If we're recovering from a problem in _make_processor, we will not
# have this attribute.
if hasattr(self, "processor") and self.processor is not None:
line = self.processor.noqa_line_for(line_number)
else:
line = None
self.results.append((error_code, line_number, column, text, line))
return error_code
def run_check(self, plugin, **arguments):
"""Run the check in a single plugin."""
LOG.debug("Running %r with %r", plugin, arguments)
assert self.processor is not None
try:
self.processor.keyword_arguments_for(
plugin["parameters"], arguments
)
except AttributeError as ae:
LOG.error("Plugin requested unknown parameters.")
raise exceptions.PluginRequestedUnknownParameters(
plugin=plugin, exception=ae
)
try:
return plugin["plugin"](**arguments)
except Exception as all_exc:
LOG.critical(
"Plugin %s raised an unexpected exception",
plugin["name"],
exc_info=True,
)
raise exceptions.PluginExecutionFailed(
plugin=plugin, exception=all_exc
)
@staticmethod
def _extract_syntax_information(exception: Exception) -> Tuple[int, int]:
if (
len(exception.args) > 1
and exception.args[1]
and len(exception.args[1]) > 2
):
token = exception.args[1]
row, column = token[1:3]
elif (
isinstance(exception, tokenize.TokenError)
and len(exception.args) == 2
and len(exception.args[1]) == 2
):
token = ()
row, column = exception.args[1]
else:
token = ()
row, column = (1, 0)
if (
column > 0
and token
and isinstance(exception, SyntaxError)
and len(token) == 4 # Python 3.9 or earlier
):
# NOTE(sigmavirus24): SyntaxErrors report 1-indexed column
# numbers. We need to decrement the column number by 1 at
# least.
column_offset = 1
row_offset = 0
# See also: https://github.com/pycqa/flake8/issues/169,
# https://github.com/PyCQA/flake8/issues/1372
# On Python 3.9 and earlier, token will be a 4-item tuple with the
# last item being the string. Starting with 3.10, they added to
# the tuple so now instead of it ending with the code that failed
# to parse, it ends with the end of the section of code that
# failed to parse. Luckily the absolute position in the tuple is
# stable across versions so we can use that here
physical_line = token[3]
# NOTE(sigmavirus24): Not all "tokens" have a string as the last
# argument. In this event, let's skip trying to find the correct
# column and row values.
if physical_line is not None:
# NOTE(sigmavirus24): SyntaxErrors also don't exactly have a
# "physical" line so much as what was accumulated by the point
# tokenizing failed.
# See also: https://github.com/pycqa/flake8/issues/169
lines = physical_line.rstrip("\n").split("\n")
row_offset = len(lines) - 1
logical_line = lines[0]
logical_line_length = len(logical_line)
if column > logical_line_length:
column = logical_line_length
row -= row_offset
column -= column_offset
return row, column
def run_ast_checks(self) -> None:
"""Run all checks expecting an abstract syntax tree."""
assert self.processor is not None
ast = self.processor.build_ast()
for plugin in self.checks["ast_plugins"]:
checker = self.run_check(plugin, tree=ast)
# If the plugin uses a class, call the run method of it, otherwise
# the call should return something iterable itself
try:
runner = checker.run()
except AttributeError:
runner = checker
for (line_number, offset, text, _) in runner:
self.report(
error_code=None,
line_number=line_number,
column=offset,
text=text,
)
def run_logical_checks(self):
"""Run all checks expecting a logical line."""
assert self.processor is not None
comments, logical_line, mapping = self.processor.build_logical_line()
if not mapping:
return
self.processor.update_state(mapping)
LOG.debug('Logical line: "%s"', logical_line.rstrip())
for plugin in self.checks["logical_line_plugins"]:
self.processor.update_checker_state_for(plugin)
results = self.run_check(plugin, logical_line=logical_line) or ()
for offset, text in results:
line_number, column_offset = find_offset(offset, mapping)
if line_number == column_offset == 0:
LOG.warning("position of error out of bounds: %s", plugin)
self.report(
error_code=None,
line_number=line_number,
column=column_offset,
text=text,
)
self.processor.next_logical_line()
def run_physical_checks(self, physical_line):
"""Run all checks for a given physical line.
A single physical check may return multiple errors.
"""
assert self.processor is not None
for plugin in self.checks["physical_line_plugins"]:
self.processor.update_checker_state_for(plugin)
result = self.run_check(plugin, physical_line=physical_line)
if result is not None:
# This is a single result if first element is an int
column_offset = None
try:
column_offset = result[0]
except (IndexError, TypeError):
pass
if isinstance(column_offset, int):
# If we only have a single result, convert to a collection
result = (result,)
for result_single in result:
column_offset, text = result_single
self.report(
error_code=None,
line_number=self.processor.line_number,
column=column_offset,
text=text,
)
def process_tokens(self):
"""Process tokens and trigger checks.
Instead of using this directly, you should use
:meth:`flake8.checker.FileChecker.run_checks`.
"""
assert self.processor is not None
parens = 0
statistics = self.statistics
file_processor = self.processor
prev_physical = ""
for token in file_processor.generate_tokens():
statistics["tokens"] += 1
self.check_physical_eol(token, prev_physical)
token_type, text = token[0:2]
processor.log_token(LOG, token)
if token_type == tokenize.OP:
parens = processor.count_parentheses(parens, text)
elif parens == 0:
if processor.token_is_newline(token):
self.handle_newline(token_type)
prev_physical = token[4]
if file_processor.tokens:
# If any tokens are left over, process them
self.run_physical_checks(file_processor.lines[-1])
self.run_logical_checks()
def run_checks(self) -> Tuple[str, Results, Dict[str, int]]:
"""Run checks against the file."""
assert self.processor is not None
try:
self.run_ast_checks()
self.process_tokens()
except (SyntaxError, tokenize.TokenError) as e:
code = "E902" if isinstance(e, tokenize.TokenError) else "E999"
row, column = self._extract_syntax_information(e)
self.report(code, row, column, f"{type(e).__name__}: {e.args[0]}")
return self.filename, self.results, self.statistics
logical_lines = self.processor.statistics["logical lines"]
self.statistics["logical lines"] = logical_lines
return self.filename, self.results, self.statistics
def handle_newline(self, token_type):
"""Handle the logic when encountering a newline token."""
assert self.processor is not None
if token_type == tokenize.NEWLINE:
self.run_logical_checks()
self.processor.reset_blank_before()
elif len(self.processor.tokens) == 1:
# The physical line contains only this token.
self.processor.visited_new_blank_line()
self.processor.delete_first_token()
else:
self.run_logical_checks()
def check_physical_eol(
self, token: processor._Token, prev_physical: str
) -> None:
"""Run physical checks if and only if it is at the end of the line."""
assert self.processor is not None
# a newline token ends a single physical line.
if processor.is_eol_token(token):
# if the file does not end with a newline, the NEWLINE
# token is inserted by the parser, but it does not contain
# the previous physical line in `token[4]`
if token[4] == "":
self.run_physical_checks(prev_physical)
else:
self.run_physical_checks(token[4])
elif processor.is_multiline_string(token):
# Less obviously, a string that contains newlines is a
# multiline string, either triple-quoted or with internal
# newlines backslash-escaped. Check every physical line in the
# string *except* for the last one: its newline is outside of
# the multiline string, so we consider it a regular physical
# line, and will check it like any other physical line.
#
# Subtleties:
# - have to wind self.line_number back because initially it
# points to the last line of the string, and we want
# check_physical() to give accurate feedback
line_no = token[2][0]
with self.processor.inside_multiline(line_number=line_no):
for line in self.processor.split_line(token):
self.run_physical_checks(line + "\n")
def _pool_init() -> None:
"""Ensure correct signaling of ^C using multiprocessing.Pool."""
signal.signal(signal.SIGINT, signal.SIG_IGN)
def _try_initialize_processpool(
job_count: int,
) -> Optional[multiprocessing.pool.Pool]:
"""Return a new process pool instance if we are able to create one."""
try:
return multiprocessing.Pool(job_count, _pool_init)
except OSError as err:
if err.errno not in SERIAL_RETRY_ERRNOS:
raise
except ImportError:
pass
return None
def calculate_pool_chunksize(num_checkers, num_jobs):
"""Determine the chunksize for the multiprocessing Pool.
- For chunksize, see: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap # noqa
- This formula, while not perfect, aims to give each worker two batches of
work.
- See: https://github.com/pycqa/flake8/issues/829#note_18878876
- See: https://github.com/pycqa/flake8/issues/197
"""
return max(num_checkers // (num_jobs * 2), 1)
def _run_checks(checker):
return checker.run_checks()
def find_offset(
offset: int, mapping: processor._LogicalMapping
) -> Tuple[int, int]:
"""Find the offset tuple for a single offset."""
if isinstance(offset, tuple):
return offset
for token in mapping:
token_offset = token[0]
if offset <= token_offset:
position = token[1]
break
else:
position = (0, 0)
offset = token_offset = 0
return (position[0], position[1] + offset - token_offset)

View file

@ -0,0 +1,44 @@
"""Constants that define defaults."""
import re
EXCLUDE = (
".svn",
"CVS",
".bzr",
".hg",
".git",
"__pycache__",
".tox",
".eggs",
"*.egg",
)
IGNORE = ("E121", "E123", "E126", "E226", "E24", "E704", "W503", "W504")
SELECT = ("E", "F", "W", "C90")
MAX_LINE_LENGTH = 79
INDENT_SIZE = 4
TRUTHY_VALUES = {"true", "1", "t"}
# Other constants
WHITESPACE = frozenset(" \t")
STATISTIC_NAMES = ("logical lines", "physical lines", "tokens")
NOQA_INLINE_REGEXP = re.compile(
# We're looking for items that look like this:
# ``# noqa``
# ``# noqa: E123``
# ``# noqa: E123,W451,F921``
# ``# noqa:E123,W451,F921``
# ``# NoQA: E123,W451,F921``
# ``# NOQA: E123,W451,F921``
# ``# NOQA:E123,W451,F921``
# We do not want to capture the ``: `` that follows ``noqa``
# We do not care about the casing of ``noqa``
# We want a comma-separated list of errors
# https://regex101.com/r/4XUuax/2 full explanation of the regex
r"# noqa(?::[\s]?(?P<codes>([A-Z]+[0-9]+(?:[,\s]+)?)+))?",
re.IGNORECASE,
)
NOQA_FILE = re.compile(r"\s*# flake8[:=]\s*noqa", re.I)

View file

@ -0,0 +1,71 @@
"""Exception classes for all of Flake8."""
from typing import Dict
class Flake8Exception(Exception):
"""Plain Flake8 exception."""
class EarlyQuit(Flake8Exception):
"""Except raised when encountering a KeyboardInterrupt."""
class ExecutionError(Flake8Exception):
"""Exception raised during execution of Flake8."""
class FailedToLoadPlugin(Flake8Exception):
"""Exception raised when a plugin fails to load."""
FORMAT = 'Flake8 failed to load plugin "%(name)s" due to %(exc)s.'
def __init__(self, plugin_name: str, exception: Exception) -> None:
"""Initialize our FailedToLoadPlugin exception."""
self.plugin_name = plugin_name
self.original_exception = exception
super().__init__(plugin_name, exception)
def __str__(self) -> str:
"""Format our exception message."""
return self.FORMAT % {
"name": self.plugin_name,
"exc": self.original_exception,
}
class PluginRequestedUnknownParameters(Flake8Exception):
"""The plugin requested unknown parameters."""
FORMAT = '"%(name)s" requested unknown parameters causing %(exc)s'
def __init__(self, plugin: Dict[str, str], exception: Exception) -> None:
"""Pop certain keyword arguments for initialization."""
self.plugin = plugin
self.original_exception = exception
super().__init__(plugin, exception)
def __str__(self) -> str:
"""Format our exception message."""
return self.FORMAT % {
"name": self.plugin["plugin_name"],
"exc": self.original_exception,
}
class PluginExecutionFailed(Flake8Exception):
"""The plugin failed during execution."""
FORMAT = '"%(name)s" failed during execution due to "%(exc)s"'
def __init__(self, plugin: Dict[str, str], exception: Exception) -> None:
"""Utilize keyword arguments for message generation."""
self.plugin = plugin
self.original_exception = exception
super().__init__(plugin, exception)
def __str__(self) -> str:
"""Format our exception message."""
return self.FORMAT % {
"name": self.plugin["plugin_name"],
"exc": self.original_exception,
}

View file

@ -0,0 +1 @@
"""Submodule containing the default formatters for Flake8."""

View file

@ -0,0 +1,211 @@
"""The base class and interface for all formatting plugins."""
import argparse
import os
import sys
from typing import IO
from typing import List
from typing import Optional
from typing import Tuple
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from flake8.statistics import Statistics
from flake8.style_guide import Violation
class BaseFormatter:
"""Class defining the formatter interface.
.. attribute:: options
The options parsed from both configuration files and the command-line.
.. attribute:: filename
If specified by the user, the path to store the results of the run.
.. attribute:: output_fd
Initialized when the :meth:`start` is called. This will be a file
object opened for writing.
.. attribute:: newline
The string to add to the end of a line. This is only used when the
output filename has been specified.
"""
def __init__(self, options: argparse.Namespace) -> None:
"""Initialize with the options parsed from config and cli.
This also calls a hook, :meth:`after_init`, so subclasses do not need
to call super to call this method.
:param options:
User specified configuration parsed from both configuration files
and the command-line interface.
:type options:
:class:`argparse.Namespace`
"""
self.options = options
self.filename = options.output_file
self.output_fd: Optional[IO[str]] = None
self.newline = "\n"
self.after_init()
def after_init(self) -> None:
"""Initialize the formatter further."""
def beginning(self, filename: str) -> None:
"""Notify the formatter that we're starting to process a file.
:param str filename:
The name of the file that Flake8 is beginning to report results
from.
"""
def finished(self, filename: str) -> None:
"""Notify the formatter that we've finished processing a file.
:param str filename:
The name of the file that Flake8 has finished reporting results
from.
"""
def start(self) -> None:
"""Prepare the formatter to receive input.
This defaults to initializing :attr:`output_fd` if :attr:`filename`
"""
if self.filename:
dirname = os.path.dirname(os.path.abspath(self.filename))
os.makedirs(dirname, exist_ok=True)
self.output_fd = open(self.filename, "a")
def handle(self, error: "Violation") -> None:
"""Handle an error reported by Flake8.
This defaults to calling :meth:`format`, :meth:`show_source`, and
then :meth:`write`. To extend how errors are handled, override this
method.
:param error:
This will be an instance of
:class:`~flake8.style_guide.Violation`.
:type error:
flake8.style_guide.Violation
"""
line = self.format(error)
source = self.show_source(error)
self.write(line, source)
def format(self, error: "Violation") -> Optional[str]:
"""Format an error reported by Flake8.
This method **must** be implemented by subclasses.
:param error:
This will be an instance of
:class:`~flake8.style_guide.Violation`.
:type error:
flake8.style_guide.Violation
:returns:
The formatted error string.
:rtype:
str
"""
raise NotImplementedError(
"Subclass of BaseFormatter did not implement" " format."
)
def show_statistics(self, statistics: "Statistics") -> None:
"""Format and print the statistics."""
for error_code in statistics.error_codes():
stats_for_error_code = statistics.statistics_for(error_code)
statistic = next(stats_for_error_code)
count = statistic.count
count += sum(stat.count for stat in stats_for_error_code)
self._write(f"{count:<5} {error_code} {statistic.message}")
def show_benchmarks(self, benchmarks: List[Tuple[str, float]]) -> None:
"""Format and print the benchmarks."""
# NOTE(sigmavirus24): The format strings are a little confusing, even
# to me, so here's a quick explanation:
# We specify the named value first followed by a ':' to indicate we're
# formatting the value.
# Next we use '<' to indicate we want the value left aligned.
# Then '10' is the width of the area.
# For floats, finally, we only want only want at most 3 digits after
# the decimal point to be displayed. This is the precision and it
# can not be specified for integers which is why we need two separate
# format strings.
float_format = "{value:<10.3} {statistic}".format
int_format = "{value:<10} {statistic}".format
for statistic, value in benchmarks:
if isinstance(value, int):
benchmark = int_format(statistic=statistic, value=value)
else:
benchmark = float_format(statistic=statistic, value=value)
self._write(benchmark)
def show_source(self, error: "Violation") -> Optional[str]:
"""Show the physical line generating the error.
This also adds an indicator for the particular part of the line that
is reported as generating the problem.
:param error:
This will be an instance of
:class:`~flake8.style_guide.Violation`.
:type error:
flake8.style_guide.Violation
:returns:
The formatted error string if the user wants to show the source.
If the user does not want to show the source, this will return
``None``.
:rtype:
str
"""
if not self.options.show_source or error.physical_line is None:
return ""
# Because column numbers are 1-indexed, we need to remove one to get
# the proper number of space characters.
indent = "".join(
c if c.isspace() else " "
for c in error.physical_line[: error.column_number - 1]
)
# Physical lines have a newline at the end, no need to add an extra
# one
return f"{error.physical_line}{indent}^"
def _write(self, output: str) -> None:
"""Handle logic of whether to use an output file or print()."""
if self.output_fd is not None:
self.output_fd.write(output + self.newline)
if self.output_fd is None or self.options.tee:
sys.stdout.buffer.write(output.encode() + self.newline.encode())
def write(self, line: Optional[str], source: Optional[str]) -> None:
"""Write the line either to the output file or stdout.
This handles deciding whether to write to a file or print to standard
out for subclasses. Override this if you want behaviour that differs
from the default.
:param str line:
The formatted string to print or write.
:param str source:
The source code that has been formatted and associated with the
line of output.
"""
if line:
self._write(line)
if source:
self._write(source)
def stop(self) -> None:
"""Clean up after reporting is finished."""
if self.output_fd is not None:
self.output_fd.close()
self.output_fd = None

View file

@ -0,0 +1,94 @@
"""Default formatting class for Flake8."""
from typing import Optional
from typing import Set
from typing import TYPE_CHECKING
from flake8.formatting import base
if TYPE_CHECKING:
from flake8.style_guide import Violation
class SimpleFormatter(base.BaseFormatter):
"""Simple abstraction for Default and Pylint formatter commonality.
Sub-classes of this need to define an ``error_format`` attribute in order
to succeed. The ``format`` method relies on that attribute and expects the
``error_format`` string to use the old-style formatting strings with named
parameters:
* code
* text
* path
* row
* col
"""
error_format: str
def format(self, error: "Violation") -> Optional[str]:
"""Format and write error out.
If an output filename is specified, write formatted errors to that
file. Otherwise, print the formatted error to standard out.
"""
return self.error_format % {
"code": error.code,
"text": error.text,
"path": error.filename,
"row": error.line_number,
"col": error.column_number,
}
class Default(SimpleFormatter):
"""Default formatter for Flake8.
This also handles backwards compatibility for people specifying a custom
format string.
"""
error_format = "%(path)s:%(row)d:%(col)d: %(code)s %(text)s"
def after_init(self) -> None:
"""Check for a custom format string."""
if self.options.format.lower() != "default":
self.error_format = self.options.format
class Pylint(SimpleFormatter):
"""Pylint formatter for Flake8."""
error_format = "%(path)s:%(row)d: [%(code)s] %(text)s"
class FilenameOnly(SimpleFormatter):
"""Only print filenames, e.g., flake8 -q."""
error_format = "%(path)s"
def after_init(self) -> None:
"""Initialize our set of filenames."""
self.filenames_already_printed: Set[str] = set()
def show_source(self, error: "Violation") -> Optional[str]:
"""Do not include the source code."""
def format(self, error: "Violation") -> Optional[str]:
"""Ensure we only print each error once."""
if error.filename not in self.filenames_already_printed:
self.filenames_already_printed.add(error.filename)
return super().format(error)
else:
return None
class Nothing(base.BaseFormatter):
"""Print absolutely nothing."""
def format(self, error: "Violation") -> Optional[str]:
"""Do nothing."""
def show_source(self, error: "Violation") -> Optional[str]:
"""Do not print the source."""

View file

@ -0,0 +1 @@
"""Module containing the logic for the Flake8 entry-points."""

View file

@ -0,0 +1,388 @@
"""Module containing the application logic for Flake8."""
import argparse
import logging
import sys
import time
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Type
from typing import TYPE_CHECKING
import flake8
from flake8 import checker
from flake8 import defaults
from flake8 import exceptions
from flake8 import style_guide
from flake8 import utils
from flake8.main import options
from flake8.options import aggregator
from flake8.options import config
from flake8.options import manager
from flake8.plugins import manager as plugin_manager
if TYPE_CHECKING:
from flake8.formatting.base import BaseFormatter
LOG = logging.getLogger(__name__)
class Application:
"""Abstract our application into a class."""
def __init__(self, program="flake8", version=flake8.__version__):
"""Initialize our application.
:param str program:
The name of the program/application that we're executing.
:param str version:
The version of the program/application we're executing.
"""
#: The timestamp when the Application instance was instantiated.
self.start_time = time.time()
#: The timestamp when the Application finished reported errors.
self.end_time: Optional[float] = None
#: The name of the program being run
self.program = program
#: The version of the program being run
self.version = version
#: The prelimary argument parser for handling options required for
#: obtaining and parsing the configuration file.
self.prelim_arg_parser = argparse.ArgumentParser(add_help=False)
options.register_preliminary_options(self.prelim_arg_parser)
#: The instance of :class:`flake8.options.manager.OptionManager` used
#: to parse and handle the options and arguments passed by the user
self.option_manager = manager.OptionManager(
prog="flake8",
version=flake8.__version__,
parents=[self.prelim_arg_parser],
)
options.register_default_options(self.option_manager)
#: The instance of :class:`flake8.plugins.manager.Checkers`
self.check_plugins: Optional[plugin_manager.Checkers] = None
#: The instance of :class:`flake8.plugins.manager.ReportFormatters`
self.formatting_plugins: Optional[
plugin_manager.ReportFormatters
] = None
#: The user-selected formatter from :attr:`formatting_plugins`
self.formatter: Optional[BaseFormatter] = None
#: The :class:`flake8.style_guide.StyleGuideManager` built from the
#: user's options
self.guide: Optional[style_guide.StyleGuideManager] = None
#: The :class:`flake8.checker.Manager` that will handle running all of
#: the checks selected by the user.
self.file_checker_manager: Optional[checker.Manager] = None
#: The user-supplied options parsed into an instance of
#: :class:`argparse.Namespace`
self.options: Optional[argparse.Namespace] = None
#: The left over arguments that were not parsed by
#: :attr:`option_manager`
self.args: Optional[List[str]] = None
#: The number of errors, warnings, and other messages after running
#: flake8 and taking into account ignored errors and lines.
self.result_count = 0
#: The total number of errors before accounting for ignored errors and
#: lines.
self.total_result_count = 0
#: Whether or not something catastrophic happened and we should exit
#: with a non-zero status code
self.catastrophic_failure = False
#: Whether the program is processing a diff or not
self.running_against_diff = False
#: The parsed diff information
self.parsed_diff: Dict[str, Set[int]] = {}
def parse_preliminary_options(
self, argv: List[str]
) -> Tuple[argparse.Namespace, List[str]]:
"""Get preliminary options from the CLI, pre-plugin-loading.
We need to know the values of a few standard options so that we can
locate configuration files and configure logging.
Since plugins aren't loaded yet, there may be some as-yet-unknown
options; we ignore those for now, they'll be parsed later when we do
real option parsing.
:param list argv:
Command-line arguments passed in directly.
:returns:
Populated namespace and list of remaining argument strings.
:rtype:
(argparse.Namespace, list)
"""
args, rest = self.prelim_arg_parser.parse_known_args(argv)
# XXX (ericvw): Special case "forwarding" the output file option so
# that it can be reparsed again for the BaseFormatter.filename.
if args.output_file:
rest.extend(("--output-file", args.output_file))
return args, rest
def exit(self) -> None:
"""Handle finalization and exiting the program.
This should be the last thing called on the application instance. It
will check certain options and exit appropriately.
"""
assert self.options is not None
if self.options.count:
print(self.result_count)
if self.options.exit_zero:
raise SystemExit(self.catastrophic_failure)
else:
raise SystemExit(
(self.result_count > 0) or self.catastrophic_failure
)
def find_plugins(self, config_finder: config.ConfigFileFinder) -> None:
"""Find and load the plugins for this application.
Set the :attr:`check_plugins` and :attr:`formatting_plugins` attributes
based on the discovered plugins found.
:param config.ConfigFileFinder config_finder:
The finder for finding and reading configuration files.
"""
local_plugins = config.get_local_plugins(config_finder)
sys.path.extend(local_plugins.paths)
self.check_plugins = plugin_manager.Checkers(local_plugins.extension)
self.formatting_plugins = plugin_manager.ReportFormatters(
local_plugins.report
)
self.check_plugins.load_plugins()
self.formatting_plugins.load_plugins()
def register_plugin_options(self) -> None:
"""Register options provided by plugins to our option manager."""
assert self.check_plugins is not None
self.check_plugins.register_options(self.option_manager)
self.check_plugins.register_plugin_versions(self.option_manager)
assert self.formatting_plugins is not None
self.formatting_plugins.register_options(self.option_manager)
def parse_configuration_and_cli(
self,
config_finder: config.ConfigFileFinder,
argv: List[str],
) -> None:
"""Parse configuration files and the CLI options.
:param config.ConfigFileFinder config_finder:
The finder for finding and reading configuration files.
:param list argv:
Command-line arguments passed in directly.
"""
self.options, self.args = aggregator.aggregate_options(
self.option_manager,
config_finder,
argv,
)
self.running_against_diff = self.options.diff
if self.running_against_diff:
self.parsed_diff = utils.parse_unified_diff()
if not self.parsed_diff:
self.exit()
assert self.check_plugins is not None
self.check_plugins.provide_options(
self.option_manager, self.options, self.args
)
assert self.formatting_plugins is not None
self.formatting_plugins.provide_options(
self.option_manager, self.options, self.args
)
def formatter_for(self, formatter_plugin_name):
"""Retrieve the formatter class by plugin name."""
assert self.formatting_plugins is not None
default_formatter = self.formatting_plugins["default"]
formatter_plugin = self.formatting_plugins.get(formatter_plugin_name)
if formatter_plugin is None:
LOG.warning(
'"%s" is an unknown formatter. Falling back to default.',
formatter_plugin_name,
)
formatter_plugin = default_formatter
return formatter_plugin.execute
def make_formatter(
self, formatter_class: Optional[Type["BaseFormatter"]] = None
) -> None:
"""Initialize a formatter based on the parsed options."""
assert self.options is not None
format_plugin = self.options.format
if 1 <= self.options.quiet < 2:
format_plugin = "quiet-filename"
elif 2 <= self.options.quiet:
format_plugin = "quiet-nothing"
if formatter_class is None:
formatter_class = self.formatter_for(format_plugin)
self.formatter = formatter_class(self.options)
def make_guide(self) -> None:
"""Initialize our StyleGuide."""
assert self.formatter is not None
assert self.options is not None
self.guide = style_guide.StyleGuideManager(
self.options, self.formatter
)
if self.running_against_diff:
self.guide.add_diff_ranges(self.parsed_diff)
def make_file_checker_manager(self) -> None:
"""Initialize our FileChecker Manager."""
self.file_checker_manager = checker.Manager(
style_guide=self.guide,
arguments=self.args,
checker_plugins=self.check_plugins,
)
def run_checks(self, files: Optional[List[str]] = None) -> None:
"""Run the actual checks with the FileChecker Manager.
This method encapsulates the logic to make a
:class:`~flake8.checker.Manger` instance run the checks it is
managing.
:param list files:
List of filenames to process
"""
assert self.file_checker_manager is not None
if self.running_against_diff:
files = sorted(self.parsed_diff)
self.file_checker_manager.start(files)
try:
self.file_checker_manager.run()
except exceptions.PluginExecutionFailed as plugin_failed:
print(str(plugin_failed))
print("Run flake8 with greater verbosity to see more details")
self.catastrophic_failure = True
LOG.info("Finished running")
self.file_checker_manager.stop()
self.end_time = time.time()
def report_benchmarks(self):
"""Aggregate, calculate, and report benchmarks for this run."""
assert self.options is not None
if not self.options.benchmark:
return
assert self.file_checker_manager is not None
assert self.end_time is not None
time_elapsed = self.end_time - self.start_time
statistics = [("seconds elapsed", time_elapsed)]
add_statistic = statistics.append
for statistic in defaults.STATISTIC_NAMES + ("files",):
value = self.file_checker_manager.statistics[statistic]
total_description = f"total {statistic} processed"
add_statistic((total_description, value))
per_second_description = f"{statistic} processed per second"
add_statistic((per_second_description, int(value / time_elapsed)))
assert self.formatter is not None
self.formatter.show_benchmarks(statistics)
def report_errors(self) -> None:
"""Report all the errors found by flake8 3.0.
This also updates the :attr:`result_count` attribute with the total
number of errors, warnings, and other messages found.
"""
LOG.info("Reporting errors")
assert self.file_checker_manager is not None
results = self.file_checker_manager.report()
self.total_result_count, self.result_count = results
LOG.info(
"Found a total of %d violations and reported %d",
self.total_result_count,
self.result_count,
)
def report_statistics(self):
"""Aggregate and report statistics from this run."""
assert self.options is not None
if not self.options.statistics:
return
assert self.formatter is not None
assert self.guide is not None
self.formatter.show_statistics(self.guide.stats)
def initialize(self, argv: List[str]) -> None:
"""Initialize the application to be run.
This finds the plugins, registers their options, and parses the
command-line arguments.
"""
# NOTE(sigmavirus24): When updating this, make sure you also update
# our legacy API calls to these same methods.
prelim_opts, remaining_args = self.parse_preliminary_options(argv)
flake8.configure_logging(prelim_opts.verbose, prelim_opts.output_file)
config_finder = config.ConfigFileFinder(
self.program,
prelim_opts.append_config,
config_file=prelim_opts.config,
ignore_config_files=prelim_opts.isolated,
)
self.find_plugins(config_finder)
self.register_plugin_options()
self.parse_configuration_and_cli(
config_finder,
remaining_args,
)
self.make_formatter()
self.make_guide()
self.make_file_checker_manager()
def report(self):
"""Report errors, statistics, and benchmarks."""
assert self.formatter is not None
self.formatter.start()
self.report_errors()
self.report_statistics()
self.report_benchmarks()
self.formatter.stop()
def _run(self, argv: List[str]) -> None:
self.initialize(argv)
self.run_checks()
self.report()
def run(self, argv: List[str]) -> None:
"""Run our application.
This method will also handle KeyboardInterrupt exceptions for the
entirety of the flake8 application. If it sees a KeyboardInterrupt it
will forcibly clean up the :class:`~flake8.checker.Manager`.
"""
try:
self._run(argv)
except KeyboardInterrupt as exc:
print("... stopped")
LOG.critical("Caught keyboard interrupt from user")
LOG.exception(exc)
self.catastrophic_failure = True
except exceptions.ExecutionError as exc:
print("There was a critical error during execution of Flake8:")
print(exc)
LOG.exception(exc)
self.catastrophic_failure = True
except exceptions.EarlyQuit:
self.catastrophic_failure = True
print("... stopped while processing files")

View file

@ -0,0 +1,23 @@
"""Command-line implementation of flake8."""
import sys
from typing import List
from typing import Optional
from flake8.main import application
def main(argv: Optional[List[str]] = None) -> None:
"""Execute the main bit of the application.
This handles the creation of an instance of :class:`Application`, runs it,
and then exits the application.
:param list argv:
The arguments to be passed to the application for parsing.
"""
if argv is None:
argv = sys.argv[1:]
app = application.Application()
app.run(argv)
app.exit()

View file

@ -0,0 +1,64 @@
"""Module containing the logic for our debugging logic."""
import argparse
import json
import platform
from typing import Dict
from typing import List
class DebugAction(argparse.Action):
"""argparse action to print debug information."""
def __init__(self, *args, **kwargs):
"""Initialize the action.
This takes an extra `option_manager` keyword argument which will be
used to delay response.
"""
self._option_manager = kwargs.pop("option_manager")
super().__init__(*args, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
"""Perform the argparse action for printing debug information."""
# NOTE(sigmavirus24): Flake8 parses options twice. The first time, we
# will not have any registered plugins. We can skip this one and only
# take action on the second time we're called.
if not self._option_manager.registered_plugins:
return
print(
json.dumps(
information(self._option_manager), indent=2, sort_keys=True
)
)
raise SystemExit(0)
def information(option_manager):
"""Generate the information to be printed for the bug report."""
return {
"version": option_manager.version,
"plugins": plugins_from(option_manager),
"dependencies": dependencies(),
"platform": {
"python_implementation": platform.python_implementation(),
"python_version": platform.python_version(),
"system": platform.system(),
},
}
def plugins_from(option_manager):
"""Generate the list of plugins installed."""
return [
{
"plugin": plugin.name,
"version": plugin.version,
"is_local": plugin.local,
}
for plugin in sorted(option_manager.registered_plugins)
]
def dependencies() -> List[Dict[str, str]]:
"""Generate the list of dependencies we care about."""
return []

View file

@ -0,0 +1,374 @@
"""Contains the logic for all of the default options for Flake8."""
import argparse
import functools
from flake8 import defaults
from flake8.main import debug
def register_preliminary_options(parser: argparse.ArgumentParser) -> None:
"""Register the preliminary options on our OptionManager.
The preliminary options include:
- ``-v``/``--verbose``
- ``--output-file``
- ``--append-config``
- ``--config``
- ``--isolated``
"""
add_argument = parser.add_argument
add_argument(
"-v",
"--verbose",
default=0,
action="count",
help="Print more information about what is happening in flake8."
" This option is repeatable and will increase verbosity each "
"time it is repeated.",
)
add_argument(
"--output-file", default=None, help="Redirect report to a file."
)
# Config file options
add_argument(
"--append-config",
action="append",
help="Provide extra config files to parse in addition to the files "
"found by Flake8 by default. These files are the last ones read "
"and so they take the highest precedence when multiple files "
"provide the same option.",
)
add_argument(
"--config",
default=None,
help="Path to the config file that will be the authoritative config "
"source. This will cause Flake8 to ignore all other "
"configuration files.",
)
add_argument(
"--isolated",
default=False,
action="store_true",
help="Ignore all configuration files.",
)
class JobsArgument:
"""Type callback for the --jobs argument."""
def __init__(self, arg: str) -> None:
"""Parse and validate the --jobs argument.
:param str arg:
The argument passed by argparse for validation
"""
self.is_auto = False
self.n_jobs = -1
if arg == "auto":
self.is_auto = True
elif arg.isdigit():
self.n_jobs = int(arg)
else:
raise argparse.ArgumentTypeError(
f"{arg!r} must be 'auto' or an integer.",
)
def __str__(self):
"""Format our JobsArgument class."""
return "auto" if self.is_auto else str(self.n_jobs)
def register_default_options(option_manager):
"""Register the default options on our OptionManager.
The default options include:
- ``-q``/``--quiet``
- ``--count``
- ``--diff``
- ``--exclude``
- ``--extend-exclude``
- ``--filename``
- ``--format``
- ``--hang-closing``
- ``--ignore``
- ``--extend-ignore``
- ``--per-file-ignores``
- ``--max-line-length``
- ``--max-doc-length``
- ``--indent-size``
- ``--select``
- ``--extend-select``
- ``--disable-noqa``
- ``--show-source``
- ``--statistics``
- ``--enable-extensions``
- ``--exit-zero``
- ``-j``/``--jobs``
- ``--tee``
- ``--benchmark``
- ``--bug-report``
"""
add_option = option_manager.add_option
# pep8 options
add_option(
"-q",
"--quiet",
default=0,
action="count",
parse_from_config=True,
help="Report only file names, or nothing. This option is repeatable.",
)
add_option(
"--count",
action="store_true",
parse_from_config=True,
help="Print total number of errors and warnings to standard error and"
" set the exit code to 1 if total is not empty.",
)
add_option(
"--diff",
action="store_true",
help="Report changes only within line number ranges in the unified "
"diff provided on standard in by the user.",
)
add_option(
"--exclude",
metavar="patterns",
default=",".join(defaults.EXCLUDE),
comma_separated_list=True,
parse_from_config=True,
normalize_paths=True,
help="Comma-separated list of files or directories to exclude."
" (Default: %(default)s)",
)
add_option(
"--extend-exclude",
metavar="patterns",
default="",
parse_from_config=True,
comma_separated_list=True,
normalize_paths=True,
help="Comma-separated list of files or directories to add to the list"
" of excluded ones.",
)
add_option(
"--filename",
metavar="patterns",
default="*.py",
parse_from_config=True,
comma_separated_list=True,
help="Only check for filenames matching the patterns in this comma-"
"separated list. (Default: %(default)s)",
)
add_option(
"--stdin-display-name",
default="stdin",
help="The name used when reporting errors from code passed via stdin."
" This is useful for editors piping the file contents to flake8."
" (Default: %(default)s)",
)
# TODO(sigmavirus24): Figure out --first/--repeat
# NOTE(sigmavirus24): We can't use choices for this option since users can
# freely provide a format string and that will break if we restrict their
# choices.
add_option(
"--format",
metavar="format",
default="default",
parse_from_config=True,
help="Format errors according to the chosen formatter.",
)
add_option(
"--hang-closing",
action="store_true",
parse_from_config=True,
help="Hang closing bracket instead of matching indentation of opening"
" bracket's line.",
)
add_option(
"--ignore",
metavar="errors",
default=",".join(defaults.IGNORE),
parse_from_config=True,
comma_separated_list=True,
help="Comma-separated list of errors and warnings to ignore (or skip)."
" For example, ``--ignore=E4,E51,W234``. (Default: %(default)s)",
)
add_option(
"--extend-ignore",
metavar="errors",
default="",
parse_from_config=True,
comma_separated_list=True,
help="Comma-separated list of errors and warnings to add to the list"
" of ignored ones. For example, ``--extend-ignore=E4,E51,W234``.",
)
add_option(
"--per-file-ignores",
default="",
parse_from_config=True,
help="A pairing of filenames and violation codes that defines which "
"violations to ignore in a particular file. The filenames can be "
"specified in a manner similar to the ``--exclude`` option and the "
"violations work similarly to the ``--ignore`` and ``--select`` "
"options.",
)
add_option(
"--max-line-length",
type=int,
metavar="n",
default=defaults.MAX_LINE_LENGTH,
parse_from_config=True,
help="Maximum allowed line length for the entirety of this run. "
"(Default: %(default)s)",
)
add_option(
"--max-doc-length",
type=int,
metavar="n",
default=None,
parse_from_config=True,
help="Maximum allowed doc line length for the entirety of this run. "
"(Default: %(default)s)",
)
add_option(
"--indent-size",
type=int,
metavar="n",
default=defaults.INDENT_SIZE,
parse_from_config=True,
help="Number of spaces used for indentation (Default: %(default)s)",
)
add_option(
"--select",
metavar="errors",
default=",".join(defaults.SELECT),
parse_from_config=True,
comma_separated_list=True,
help="Comma-separated list of errors and warnings to enable."
" For example, ``--select=E4,E51,W234``. (Default: %(default)s)",
)
add_option(
"--extend-select",
metavar="errors",
default="",
parse_from_config=True,
comma_separated_list=True,
help=(
"Comma-separated list of errors and warnings to add to the list "
"of selected ones. For example, ``--extend-select=E4,E51,W234``."
),
)
add_option(
"--disable-noqa",
default=False,
parse_from_config=True,
action="store_true",
help='Disable the effect of "# noqa". This will report errors on '
'lines with "# noqa" at the end.',
)
# TODO(sigmavirus24): Decide what to do about --show-pep8
add_option(
"--show-source",
action="store_true",
parse_from_config=True,
help="Show the source generate each error or warning.",
)
add_option(
"--no-show-source",
action="store_false",
dest="show_source",
parse_from_config=False,
help="Negate --show-source",
)
add_option(
"--statistics",
action="store_true",
parse_from_config=True,
help="Count errors and warnings.",
)
# Flake8 options
add_option(
"--enable-extensions",
default="",
parse_from_config=True,
comma_separated_list=True,
help="Enable plugins and extensions that are otherwise disabled "
"by default",
)
add_option(
"--exit-zero",
action="store_true",
help='Exit with status code "0" even if there are errors.',
)
add_option(
"-j",
"--jobs",
default="auto",
parse_from_config=True,
type=JobsArgument,
help="Number of subprocesses to use to run checks in parallel. "
'This is ignored on Windows. The default, "auto", will '
"auto-detect the number of processors available to use."
" (Default: %(default)s)",
)
add_option(
"--tee",
default=False,
parse_from_config=True,
action="store_true",
help="Write to stdout and output-file.",
)
# Benchmarking
add_option(
"--benchmark",
default=False,
action="store_true",
help="Print benchmark information about this run of Flake8",
)
# Debugging
add_option(
"--bug-report",
action=functools.partial(
debug.DebugAction, option_manager=option_manager
),
nargs=0,
help="Print information necessary when preparing a bug report",
)

View file

@ -0,0 +1,12 @@
"""Package containing the option manager and config management logic.
- :mod:`flake8.options.config` contains the logic for finding, parsing, and
merging configuration files.
- :mod:`flake8.options.manager` contains the logic for managing customized
Flake8 command-line and configuration options.
- :mod:`flake8.options.aggregator` uses objects from both of the above modules
to aggregate configuration into one object used by plugins and Flake8.
"""

View file

@ -0,0 +1,86 @@
"""Aggregation function for CLI specified options and config file options.
This holds the logic that uses the collected and merged config files and
applies the user-specified command-line configuration on top of it.
"""
import argparse
import logging
from typing import List
from typing import Tuple
from flake8.options import config
from flake8.options.manager import OptionManager
LOG = logging.getLogger(__name__)
def aggregate_options(
manager: OptionManager,
config_finder: config.ConfigFileFinder,
argv: List[str],
) -> Tuple[argparse.Namespace, List[str]]:
"""Aggregate and merge CLI and config file options.
:param flake8.options.manager.OptionManager manager:
The instance of the OptionManager that we're presently using.
:param flake8.options.config.ConfigFileFinder config_finder:
The config file finder to use.
:param list argv:
The list of remaining command-line arguments that were unknown during
preliminary option parsing to pass to ``manager.parse_args``.
:returns:
Tuple of the parsed options and extra arguments returned by
``manager.parse_args``.
:rtype:
tuple(argparse.Namespace, list)
"""
# Get defaults from the option parser
default_values, _ = manager.parse_args([])
# Make our new configuration file mergerator
config_parser = config.ConfigParser(
option_manager=manager, config_finder=config_finder
)
# Get the parsed config
parsed_config = config_parser.parse()
# Extend the default ignore value with the extended default ignore list,
# registered by plugins.
extended_default_ignore = manager.extended_default_ignore.copy()
# Let's store our extended default ignore for use by the decision engine
default_values.extended_default_ignore = (
manager.extended_default_ignore.copy()
)
LOG.debug(
"Extended default ignore list: %s", list(extended_default_ignore)
)
extended_default_ignore.update(default_values.ignore)
default_values.ignore = list(extended_default_ignore)
LOG.debug("Merged default ignore list: %s", default_values.ignore)
extended_default_select = manager.extended_default_select.copy()
LOG.debug(
"Extended default select list: %s", list(extended_default_select)
)
default_values.extended_default_select = extended_default_select
# Merge values parsed from config onto the default values returned
for config_name, value in parsed_config.items():
dest_name = config_name
# If the config name is somehow different from the destination name,
# fetch the destination name from our Option
if not hasattr(default_values, config_name):
dest_name = config_parser.config_options[config_name].dest
LOG.debug(
'Overriding default value of (%s) for "%s" with (%s)',
getattr(default_values, dest_name, None),
dest_name,
value,
)
# Override the default values with the config values
setattr(default_values, dest_name, value)
# Finally parse the command-line options
return manager.parse_args(argv, default_values)

View file

@ -0,0 +1,318 @@
"""Config handling logic for Flake8."""
import collections
import configparser
import logging
import os.path
from typing import List
from typing import Optional
from typing import Tuple
from flake8 import utils
LOG = logging.getLogger(__name__)
__all__ = ("ConfigFileFinder", "ConfigParser")
class ConfigFileFinder:
"""Encapsulate the logic for finding and reading config files."""
def __init__(
self,
program_name: str,
extra_config_files: Optional[List[str]] = None,
config_file: Optional[str] = None,
ignore_config_files: bool = False,
) -> None:
"""Initialize object to find config files.
:param str program_name:
Name of the current program (e.g., flake8).
:param list extra_config_files:
Extra configuration files specified by the user to read.
:param str config_file:
Configuration file override to only read configuration from.
:param bool ignore_config_files:
Determine whether to ignore configuration files or not.
"""
# The values of --append-config from the CLI
if extra_config_files is None:
extra_config_files = []
self.extra_config_files = utils.normalize_paths(extra_config_files)
# The value of --config from the CLI.
self.config_file = config_file
# The value of --isolated from the CLI.
self.ignore_config_files = ignore_config_files
# User configuration file.
self.program_name = program_name
# List of filenames to find in the local/project directory
self.project_filenames = ("setup.cfg", "tox.ini", f".{program_name}")
self.local_directory = os.path.abspath(os.curdir)
@staticmethod
def _read_config(
*files: str,
) -> Tuple[configparser.RawConfigParser, List[str]]:
config = configparser.RawConfigParser()
found_files = []
for filename in files:
try:
found_files.extend(config.read(filename))
except UnicodeDecodeError:
LOG.exception(
"There was an error decoding a config file."
"The file with a problem was %s.",
filename,
)
except configparser.ParsingError:
LOG.exception(
"There was an error trying to parse a config "
"file. The file with a problem was %s.",
filename,
)
return (config, found_files)
def cli_config(self, files: str) -> configparser.RawConfigParser:
"""Read and parse the config file specified on the command-line."""
config, found_files = self._read_config(files)
if found_files:
LOG.debug("Found cli configuration files: %s", found_files)
return config
def generate_possible_local_files(self):
"""Find and generate all local config files."""
parent = tail = os.getcwd()
found_config_files = False
while tail and not found_config_files:
for project_filename in self.project_filenames:
filename = os.path.abspath(
os.path.join(parent, project_filename)
)
if os.path.exists(filename):
yield filename
found_config_files = True
self.local_directory = parent
(parent, tail) = os.path.split(parent)
def local_config_files(self):
"""Find all local config files which actually exist.
Filter results from
:meth:`~ConfigFileFinder.generate_possible_local_files` based
on whether the filename exists or not.
:returns:
List of files that exist that are local project config files with
extra config files appended to that list (which also exist).
:rtype:
[str]
"""
exists = os.path.exists
return [
filename for filename in self.generate_possible_local_files()
] + [f for f in self.extra_config_files if exists(f)]
def local_configs_with_files(self):
"""Parse all local config files into one config object.
Return (config, found_config_files) tuple.
"""
config, found_files = self._read_config(*self.local_config_files())
if found_files:
LOG.debug("Found local configuration files: %s", found_files)
return (config, found_files)
def local_configs(self):
"""Parse all local config files into one config object."""
return self.local_configs_with_files()[0]
class ConfigParser:
"""Encapsulate merging different types of configuration files.
This parses out the options registered that were specified in the
configuration files, handles extra configuration files, and returns
dictionaries with the parsed values.
"""
#: Set of actions that should use the
#: :meth:`~configparser.RawConfigParser.getbool` method.
GETBOOL_ACTIONS = {"store_true", "store_false"}
def __init__(self, option_manager, config_finder):
"""Initialize the ConfigParser instance.
:param flake8.options.manager.OptionManager option_manager:
Initialized OptionManager.
:param flake8.options.config.ConfigFileFinder config_finder:
Initialized ConfigFileFinder.
"""
#: Our instance of flake8.options.manager.OptionManager
self.option_manager = option_manager
#: The prog value for the cli parser
self.program_name = option_manager.program_name
#: Mapping of configuration option names to
#: :class:`~flake8.options.manager.Option` instances
self.config_options = option_manager.config_options_dict
#: Our instance of our :class:`~ConfigFileFinder`
self.config_finder = config_finder
def _normalize_value(self, option, value, parent=None):
if parent is None:
parent = self.config_finder.local_directory
final_value = option.normalize(value, parent)
LOG.debug(
'%r has been normalized to %r for option "%s"',
value,
final_value,
option.config_name,
)
return final_value
def _parse_config(self, config_parser, parent=None):
config_dict = {}
for option_name in config_parser.options(self.program_name):
if option_name not in self.config_options:
LOG.debug(
'Option "%s" is not registered. Ignoring.', option_name
)
continue
option = self.config_options[option_name]
# Use the appropriate method to parse the config value
method = config_parser.get
if option.type is int or option.action == "count":
method = config_parser.getint
elif option.action in self.GETBOOL_ACTIONS:
method = config_parser.getboolean
value = method(self.program_name, option_name)
LOG.debug('Option "%s" returned value: %r', option_name, value)
final_value = self._normalize_value(option, value, parent)
config_dict[option.config_name] = final_value
return config_dict
def is_configured_by(self, config):
"""Check if the specified config parser has an appropriate section."""
return config.has_section(self.program_name)
def parse_local_config(self):
"""Parse and return the local configuration files."""
config = self.config_finder.local_configs()
if not self.is_configured_by(config):
LOG.debug(
"Local configuration files have no %s section",
self.program_name,
)
return {}
LOG.debug("Parsing local configuration files.")
return self._parse_config(config)
def parse_cli_config(self, config_path):
"""Parse and return the file specified by --config."""
config = self.config_finder.cli_config(config_path)
if not self.is_configured_by(config):
LOG.debug(
"CLI configuration files have no %s section",
self.program_name,
)
return {}
LOG.debug("Parsing CLI configuration files.")
return self._parse_config(config, os.path.dirname(config_path))
def parse(self):
"""Parse and return the local config files.
:returns:
Dictionary of parsed configuration options
:rtype:
dict
"""
if self.config_finder.ignore_config_files:
LOG.debug(
"Refusing to parse configuration files due to user-"
"requested isolation"
)
return {}
if self.config_finder.config_file:
LOG.debug(
"Ignoring user and locally found configuration files. "
'Reading only configuration from "%s" specified via '
"--config by the user",
self.config_finder.config_file,
)
return self.parse_cli_config(self.config_finder.config_file)
return self.parse_local_config()
def get_local_plugins(config_finder):
"""Get local plugins lists from config files.
:param flake8.options.config.ConfigFileFinder config_finder:
The config file finder to use.
:returns:
LocalPlugins namedtuple containing two lists of plugin strings,
one for extension (checker) plugins and one for report plugins.
:rtype:
flake8.options.config.LocalPlugins
"""
local_plugins = LocalPlugins(extension=[], report=[], paths=[])
if config_finder.ignore_config_files:
LOG.debug(
"Refusing to look for local plugins in configuration"
"files due to user-requested isolation"
)
return local_plugins
if config_finder.config_file:
LOG.debug(
'Reading local plugins only from "%s" specified via '
"--config by the user",
config_finder.config_file,
)
config = config_finder.cli_config(config_finder.config_file)
config_files = [config_finder.config_file]
else:
config, config_files = config_finder.local_configs_with_files()
base_dirs = {os.path.dirname(cf) for cf in config_files}
section = f"{config_finder.program_name}:local-plugins"
for plugin_type in ["extension", "report"]:
if config.has_option(section, plugin_type):
local_plugins_string = config.get(section, plugin_type).strip()
plugin_type_list = getattr(local_plugins, plugin_type)
plugin_type_list.extend(
utils.parse_comma_separated_list(
local_plugins_string, regexp=utils.LOCAL_PLUGIN_LIST_RE
)
)
if config.has_option(section, "paths"):
raw_paths = utils.parse_comma_separated_list(
config.get(section, "paths").strip()
)
norm_paths: List[str] = []
for base_dir in base_dirs:
norm_paths.extend(
path
for path in utils.normalize_paths(raw_paths, parent=base_dir)
if os.path.exists(path)
)
local_plugins.paths.extend(norm_paths)
return local_plugins
LocalPlugins = collections.namedtuple("LocalPlugins", "extension report paths")

View file

@ -0,0 +1,525 @@
"""Option handling and Option management logic."""
import argparse
import collections
import contextlib
import enum
import functools
import logging
from typing import Any
from typing import Callable
from typing import cast
from typing import Dict
from typing import Generator
from typing import List
from typing import Mapping
from typing import Optional
from typing import Sequence
from typing import Set
from typing import Tuple
from typing import Type
from typing import TYPE_CHECKING
from typing import Union
from flake8 import utils
if TYPE_CHECKING:
from typing import NoReturn
LOG = logging.getLogger(__name__)
# represent a singleton of "not passed arguments".
# an enum is chosen to trick mypy
_ARG = enum.Enum("_ARG", "NO")
_optparse_callable_map: Dict[str, Union[Type[Any], _ARG]] = {
"int": int,
"long": int,
"string": str,
"float": float,
"complex": complex,
"choice": _ARG.NO,
# optparse allows this but does not document it
"str": str,
}
class _CallbackAction(argparse.Action):
"""Shim for optparse-style callback actions."""
def __init__(self, *args: Any, **kwargs: Any) -> None:
self._callback = kwargs.pop("callback")
self._callback_args = kwargs.pop("callback_args", ())
self._callback_kwargs = kwargs.pop("callback_kwargs", {})
super().__init__(*args, **kwargs)
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Optional[Union[Sequence[str], str]],
option_string: Optional[str] = None,
) -> None:
if not values:
values = None
elif isinstance(values, list) and len(values) > 1:
values = tuple(values)
self._callback(
self,
option_string,
values,
parser,
*self._callback_args,
**self._callback_kwargs,
)
def _flake8_normalize(
value: str, *args: str, **kwargs: bool
) -> Union[str, List[str]]:
comma_separated_list = kwargs.pop("comma_separated_list", False)
normalize_paths = kwargs.pop("normalize_paths", False)
if kwargs:
raise TypeError(f"Unexpected keyword args: {kwargs}")
ret: Union[str, List[str]] = value
if comma_separated_list and isinstance(ret, str):
ret = utils.parse_comma_separated_list(value)
if normalize_paths:
if isinstance(ret, str):
ret = utils.normalize_path(ret, *args)
else:
ret = utils.normalize_paths(ret, *args)
return ret
class Option:
"""Our wrapper around an argparse argument parsers to add features."""
def __init__(
self,
short_option_name: Union[str, _ARG] = _ARG.NO,
long_option_name: Union[str, _ARG] = _ARG.NO,
# Options below here are taken from the optparse.Option class
action: Union[str, Type[argparse.Action], _ARG] = _ARG.NO,
default: Union[Any, _ARG] = _ARG.NO,
type: Union[str, Callable[..., Any], _ARG] = _ARG.NO,
dest: Union[str, _ARG] = _ARG.NO,
nargs: Union[int, str, _ARG] = _ARG.NO,
const: Union[Any, _ARG] = _ARG.NO,
choices: Union[Sequence[Any], _ARG] = _ARG.NO,
help: Union[str, _ARG] = _ARG.NO,
metavar: Union[str, _ARG] = _ARG.NO,
# deprecated optparse-only options
callback: Union[Callable[..., Any], _ARG] = _ARG.NO,
callback_args: Union[Sequence[Any], _ARG] = _ARG.NO,
callback_kwargs: Union[Mapping[str, Any], _ARG] = _ARG.NO,
# Options below are taken from argparse.ArgumentParser.add_argument
required: Union[bool, _ARG] = _ARG.NO,
# Options below here are specific to Flake8
parse_from_config: bool = False,
comma_separated_list: bool = False,
normalize_paths: bool = False,
) -> None:
"""Initialize an Option instance.
The following are all passed directly through to argparse.
:param str short_option_name:
The short name of the option (e.g., ``-x``). This will be the
first argument passed to ``ArgumentParser.add_argument``
:param str long_option_name:
The long name of the option (e.g., ``--xtra-long-option``). This
will be the second argument passed to
``ArgumentParser.add_argument``
:param default:
Default value of the option.
:param dest:
Attribute name to store parsed option value as.
:param nargs:
Number of arguments to parse for this option.
:param const:
Constant value to store on a common destination. Usually used in
conjunction with ``action="store_const"``.
:param iterable choices:
Possible values for the option.
:param str help:
Help text displayed in the usage information.
:param str metavar:
Name to use instead of the long option name for help text.
:param bool required:
Whether this option is required or not.
The following options may be passed directly through to :mod:`argparse`
but may need some massaging.
:param type:
A callable to normalize the type (as is the case in
:mod:`argparse`). Deprecated: you can also pass through type
strings such as ``'int'`` which are handled by :mod:`optparse`.
:param str action:
Any action allowed by :mod:`argparse`. Deprecated: this also
understands the ``action='callback'`` action from :mod:`optparse`.
:param callable callback:
Callback used if the action is ``"callback"``. Deprecated: please
use ``action=`` instead.
:param iterable callback_args:
Additional positional arguments to the callback callable.
Deprecated: please use ``action=`` instead (probably with
``functools.partial``).
:param dictionary callback_kwargs:
Keyword arguments to the callback callable. Deprecated: please
use ``action=`` instead (probably with ``functools.partial``).
The following parameters are for Flake8's option handling alone.
:param bool parse_from_config:
Whether or not this option should be parsed out of config files.
:param bool comma_separated_list:
Whether the option is a comma separated list when parsing from a
config file.
:param bool normalize_paths:
Whether the option is expecting a path or list of paths and should
attempt to normalize the paths to absolute paths.
"""
if (
long_option_name is _ARG.NO
and short_option_name is not _ARG.NO
and short_option_name.startswith("--")
):
short_option_name, long_option_name = _ARG.NO, short_option_name
# optparse -> argparse `%default` => `%(default)s`
if help is not _ARG.NO and "%default" in help:
LOG.warning(
"option %s: please update `help=` text to use %%(default)s "
"instead of %%default -- this will be an error in the future",
long_option_name,
)
help = help.replace("%default", "%(default)s")
# optparse -> argparse for `callback`
if action == "callback":
LOG.warning(
"option %s: please update from optparse `action='callback'` "
"to argparse action classes -- this will be an error in the "
"future",
long_option_name,
)
action = _CallbackAction
if type is _ARG.NO:
nargs = 0
# optparse -> argparse for `type`
if isinstance(type, str):
LOG.warning(
"option %s: please update from optparse string `type=` to "
"argparse callable `type=` -- this will be an error in the "
"future",
long_option_name,
)
type = _optparse_callable_map[type]
# flake8 special type normalization
if comma_separated_list or normalize_paths:
type = functools.partial(
_flake8_normalize,
comma_separated_list=comma_separated_list,
normalize_paths=normalize_paths,
)
self.short_option_name = short_option_name
self.long_option_name = long_option_name
self.option_args = [
x
for x in (short_option_name, long_option_name)
if x is not _ARG.NO
]
self.action = action
self.default = default
self.type = type
self.dest = dest
self.nargs = nargs
self.const = const
self.choices = choices
self.callback = callback
self.callback_args = callback_args
self.callback_kwargs = callback_kwargs
self.help = help
self.metavar = metavar
self.required = required
self.option_kwargs: Dict[str, Union[Any, _ARG]] = {
"action": self.action,
"default": self.default,
"type": self.type,
"dest": self.dest,
"nargs": self.nargs,
"const": self.const,
"choices": self.choices,
"callback": self.callback,
"callback_args": self.callback_args,
"callback_kwargs": self.callback_kwargs,
"help": self.help,
"metavar": self.metavar,
"required": self.required,
}
# Set our custom attributes
self.parse_from_config = parse_from_config
self.comma_separated_list = comma_separated_list
self.normalize_paths = normalize_paths
self.config_name: Optional[str] = None
if parse_from_config:
if long_option_name is _ARG.NO:
raise ValueError(
"When specifying parse_from_config=True, "
"a long_option_name must also be specified."
)
self.config_name = long_option_name[2:].replace("-", "_")
self._opt = None
@property
def filtered_option_kwargs(self) -> Dict[str, Any]:
"""Return any actually-specified arguments."""
return {
k: v for k, v in self.option_kwargs.items() if v is not _ARG.NO
}
def __repr__(self) -> str: # noqa: D105
parts = []
for arg in self.option_args:
parts.append(arg)
for k, v in self.filtered_option_kwargs.items():
parts.append(f"{k}={v!r}")
return f"Option({', '.join(parts)})"
def normalize(self, value: Any, *normalize_args: str) -> Any:
"""Normalize the value based on the option configuration."""
if self.comma_separated_list and isinstance(value, str):
value = utils.parse_comma_separated_list(value)
if self.normalize_paths:
if isinstance(value, list):
value = utils.normalize_paths(value, *normalize_args)
else:
value = utils.normalize_path(value, *normalize_args)
return value
def normalize_from_setuptools(
self, value: str
) -> Union[int, float, complex, bool, str]:
"""Normalize the value received from setuptools."""
value = self.normalize(value)
if self.type is int or self.action == "count":
return int(value)
elif self.type is float:
return float(value)
elif self.type is complex:
return complex(value)
if self.action in ("store_true", "store_false"):
value = str(value).upper()
if value in ("1", "T", "TRUE", "ON"):
return True
if value in ("0", "F", "FALSE", "OFF"):
return False
return value
def to_argparse(self) -> Tuple[List[str], Dict[str, Any]]:
"""Convert a Flake8 Option to argparse ``add_argument`` arguments."""
return self.option_args, self.filtered_option_kwargs
@property
def to_optparse(self) -> "NoReturn":
"""No longer functional."""
raise AttributeError("to_optparse: flake8 now uses argparse")
PluginVersion = collections.namedtuple(
"PluginVersion", ["name", "version", "local"]
)
class OptionManager:
"""Manage Options and OptionParser while adding post-processing."""
def __init__(
self,
prog: str,
version: str,
usage: str = "%(prog)s [options] file file ...",
parents: Optional[List[argparse.ArgumentParser]] = None,
) -> None: # noqa: E501
"""Initialize an instance of an OptionManager.
:param str prog:
Name of the actual program (e.g., flake8).
:param str version:
Version string for the program.
:param str usage:
Basic usage string used by the OptionParser.
:param argparse.ArgumentParser parents:
A list of ArgumentParser objects whose arguments should also be
included.
"""
if parents is None:
parents = []
self.parser: argparse.ArgumentParser = argparse.ArgumentParser(
prog=prog, usage=usage, parents=parents
)
self._current_group: Optional[argparse._ArgumentGroup] = None
self.version_action = cast(
"argparse._VersionAction",
self.parser.add_argument(
"--version", action="version", version=version
),
)
self.parser.add_argument("filenames", nargs="*", metavar="filename")
self.config_options_dict: Dict[str, Option] = {}
self.options: List[Option] = []
self.program_name = prog
self.version = version
self.registered_plugins: Set[PluginVersion] = set()
self.extended_default_ignore: Set[str] = set()
self.extended_default_select: Set[str] = set()
@contextlib.contextmanager
def group(self, name: str) -> Generator[None, None, None]:
"""Attach options to an argparse group during this context."""
group = self.parser.add_argument_group(name)
self._current_group, orig_group = group, self._current_group
try:
yield
finally:
self._current_group = orig_group
def add_option(self, *args: Any, **kwargs: Any) -> None:
"""Create and register a new option.
See parameters for :class:`~flake8.options.manager.Option` for
acceptable arguments to this method.
.. note::
``short_option_name`` and ``long_option_name`` may be specified
positionally as they are with argparse normally.
"""
option = Option(*args, **kwargs)
option_args, option_kwargs = option.to_argparse()
if self._current_group is not None:
self._current_group.add_argument(*option_args, **option_kwargs)
else:
self.parser.add_argument(*option_args, **option_kwargs)
self.options.append(option)
if option.parse_from_config:
name = option.config_name
assert name is not None # nosec (for mypy)
self.config_options_dict[name] = option
self.config_options_dict[name.replace("_", "-")] = option
LOG.debug('Registered option "%s".', option)
def remove_from_default_ignore(self, error_codes: Sequence[str]) -> None:
"""Remove specified error codes from the default ignore list.
:param list error_codes:
List of strings that are the error/warning codes to attempt to
remove from the extended default ignore list.
"""
LOG.debug("Removing %r from the default ignore list", error_codes)
for error_code in error_codes:
try:
self.extended_default_ignore.remove(error_code)
except (ValueError, KeyError):
LOG.debug(
"Attempted to remove %s from default ignore"
" but it was not a member of the list.",
error_code,
)
def extend_default_ignore(self, error_codes: Sequence[str]) -> None:
"""Extend the default ignore list with the error codes provided.
:param list error_codes:
List of strings that are the error/warning codes with which to
extend the default ignore list.
"""
LOG.debug("Extending default ignore list with %r", error_codes)
self.extended_default_ignore.update(error_codes)
def extend_default_select(self, error_codes: Sequence[str]) -> None:
"""Extend the default select list with the error codes provided.
:param list error_codes:
List of strings that are the error/warning codes with which
to extend the default select list.
"""
LOG.debug("Extending default select list with %r", error_codes)
self.extended_default_select.update(error_codes)
def generate_versions(
self, format_str: str = "%(name)s: %(version)s", join_on: str = ", "
) -> str:
"""Generate a comma-separated list of versions of plugins."""
return join_on.join(
format_str % plugin._asdict()
for plugin in sorted(self.registered_plugins)
)
def update_version_string(self) -> None:
"""Update the flake8 version string."""
self.version_action.version = "{} ({}) {}".format(
self.version, self.generate_versions(), utils.get_python_version()
)
def generate_epilog(self) -> None:
"""Create an epilog with the version and name of each of plugin."""
plugin_version_format = "%(name)s: %(version)s"
self.parser.epilog = "Installed plugins: " + self.generate_versions(
plugin_version_format
)
def parse_args(
self,
args: Optional[List[str]] = None,
values: Optional[argparse.Namespace] = None,
) -> Tuple[argparse.Namespace, List[str]]:
"""Proxy to calling the OptionParser's parse_args method."""
self.generate_epilog()
self.update_version_string()
if values:
self.parser.set_defaults(**vars(values))
parsed_args = self.parser.parse_args(args)
# TODO: refactor callers to not need this
return parsed_args, parsed_args.filenames
def parse_known_args(
self, args: Optional[List[str]] = None
) -> Tuple[argparse.Namespace, List[str]]:
"""Parse only the known arguments from the argument values.
Replicate a little argparse behaviour while we're still on
optparse.
"""
self.generate_epilog()
self.update_version_string()
return self.parser.parse_known_args(args)
def register_plugin(
self, name: str, version: str, local: bool = False
) -> None:
"""Register a plugin relying on the OptionManager.
:param str name:
The name of the checker itself. This will be the ``name``
attribute of the class or function loaded from the entry-point.
:param str version:
The version of the checker that we're using.
:param bool local:
Whether the plugin is local to the project/repository or not.
"""
self.registered_plugins.add(PluginVersion(name, version, local))

View file

@ -0,0 +1 @@
"""Submodule of built-in plugins and plugin managers."""

View file

@ -0,0 +1,533 @@
"""Plugin loading and management logic and classes."""
import logging
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from flake8 import exceptions
from flake8 import utils
from flake8._compat import importlib_metadata
LOG = logging.getLogger(__name__)
__all__ = ("Checkers", "Plugin", "PluginManager", "ReportFormatters")
NO_GROUP_FOUND = object()
class Plugin:
"""Wrap an EntryPoint from setuptools and other logic."""
def __init__(self, name, entry_point, local=False):
"""Initialize our Plugin.
:param str name:
Name of the entry-point as it was registered with setuptools.
:param entry_point:
EntryPoint returned by setuptools.
:type entry_point:
setuptools.EntryPoint
:param bool local:
Is this a repo-local plugin?
"""
self.name = name
self.entry_point = entry_point
self.local = local
self._plugin: Any = None
self._parameters = None
self._parameter_names: Optional[List[str]] = None
self._group = None
self._plugin_name = None
self._version = None
def __repr__(self) -> str:
"""Provide an easy to read description of the current plugin."""
return 'Plugin(name="{}", entry_point="{}")'.format(
self.name, self.entry_point.value
)
def to_dictionary(self):
"""Convert this plugin to a dictionary."""
return {
"name": self.name,
"parameters": self.parameters,
"parameter_names": self.parameter_names,
"plugin": self.plugin,
"plugin_name": self.plugin_name,
}
def is_in_a_group(self):
"""Determine if this plugin is in a group.
:returns:
True if the plugin is in a group, otherwise False.
:rtype:
bool
"""
return self.group() is not None
def group(self):
"""Find and parse the group the plugin is in."""
if self._group is None:
name = self.name.split(".", 1)
if len(name) > 1:
self._group = name[0]
else:
self._group = NO_GROUP_FOUND
if self._group is NO_GROUP_FOUND:
return None
return self._group
@property
def parameters(self):
"""List of arguments that need to be passed to the plugin."""
if self._parameters is None:
self._parameters = utils.parameters_for(self)
return self._parameters
@property
def parameter_names(self) -> List[str]:
"""List of argument names that need to be passed to the plugin."""
if self._parameter_names is None:
self._parameter_names = list(self.parameters)
return self._parameter_names
@property
def plugin(self):
"""Load and return the plugin associated with the entry-point.
This property implicitly loads the plugin and then caches it.
"""
self.load_plugin()
return self._plugin
@property
def version(self) -> str:
"""Return the version of the plugin."""
version = self._version
if version is None:
if self.is_in_a_group():
version = self._version = version_for(self)
else:
version = self._version = self.plugin.version
return version
@property
def plugin_name(self):
"""Return the name of the plugin."""
if self._plugin_name is None:
if self.is_in_a_group():
self._plugin_name = self.group()
else:
self._plugin_name = self.plugin.name
return self._plugin_name
@property
def off_by_default(self):
"""Return whether the plugin is ignored by default."""
return getattr(self.plugin, "off_by_default", False)
def execute(self, *args, **kwargs):
r"""Call the plugin with \*args and \*\*kwargs."""
return self.plugin(*args, **kwargs) # pylint: disable=not-callable
def _load(self):
self._plugin = self.entry_point.load()
if not callable(self._plugin):
msg = (
f"Plugin {self._plugin!r} is not a callable. It might be "
f"written for an older version of flake8 and might not work "
f"with this version"
)
LOG.critical(msg)
raise TypeError(msg)
def load_plugin(self):
"""Retrieve the plugin for this entry-point.
This loads the plugin, stores it on the instance and then returns it.
It does not reload it after the first time, it merely returns the
cached plugin.
:returns:
Nothing
"""
if self._plugin is None:
LOG.info('Loading plugin "%s" from entry-point.', self.name)
try:
self._load()
except Exception as load_exception:
LOG.exception(load_exception)
failed_to_load = exceptions.FailedToLoadPlugin(
plugin_name=self.name, exception=load_exception
)
LOG.critical(str(failed_to_load))
raise failed_to_load
def enable(self, optmanager, options=None):
"""Remove plugin name from the default ignore list."""
optmanager.remove_from_default_ignore([self.name])
optmanager.extend_default_select([self.name])
if not options:
return
try:
options.ignore.remove(self.name)
except (ValueError, KeyError):
LOG.debug(
"Attempted to remove %s from the ignore list but it was "
"not a member of the list.",
self.name,
)
def disable(self, optmanager):
"""Add the plugin name to the default ignore list."""
optmanager.extend_default_ignore([self.name])
def provide_options(self, optmanager, options, extra_args):
"""Pass the parsed options and extra arguments to the plugin."""
parse_options = getattr(self.plugin, "parse_options", None)
if parse_options is not None:
LOG.debug('Providing options to plugin "%s".', self.name)
try:
parse_options(optmanager, options, extra_args)
except TypeError:
parse_options(options)
if self.name in options.enable_extensions:
self.enable(optmanager, options)
def register_options(self, optmanager):
"""Register the plugin's command-line options on the OptionManager.
:param optmanager:
Instantiated OptionManager to register options on.
:type optmanager:
flake8.options.manager.OptionManager
:returns:
Nothing
"""
add_options = getattr(self.plugin, "add_options", None)
if add_options is not None:
LOG.debug(
'Registering options from plugin "%s" on OptionManager %r',
self.name,
optmanager,
)
with optmanager.group(self.plugin_name):
add_options(optmanager)
if self.off_by_default:
self.disable(optmanager)
class PluginManager: # pylint: disable=too-few-public-methods
"""Find and manage plugins consistently."""
def __init__(
self, namespace: str, local_plugins: Optional[List[str]] = None
) -> None:
"""Initialize the manager.
:param str namespace:
Namespace of the plugins to manage, e.g., 'flake8.extension'.
:param list local_plugins:
Plugins from config (as "X = path.to:Plugin" strings).
"""
self.namespace = namespace
self.plugins: Dict[str, Plugin] = {}
self.names: List[str] = []
self._load_local_plugins(local_plugins or [])
self._load_entrypoint_plugins()
def _load_local_plugins(self, local_plugins):
"""Load local plugins from config.
:param list local_plugins:
Plugins from config (as "X = path.to:Plugin" strings).
"""
for plugin_str in local_plugins:
name, _, entry_str = plugin_str.partition("=")
name, entry_str = name.strip(), entry_str.strip()
entry_point = importlib_metadata.EntryPoint(
name, entry_str, self.namespace
)
self._load_plugin_from_entrypoint(entry_point, local=True)
def _load_entrypoint_plugins(self):
LOG.info('Loading entry-points for "%s".', self.namespace)
eps = importlib_metadata.entry_points().get(self.namespace, ())
# python2.7 occasionally gives duplicate results due to redundant
# `local/lib` -> `../lib` symlink on linux in virtualenvs so we
# eliminate duplicates here
for entry_point in sorted(frozenset(eps)):
if entry_point.name == "per-file-ignores":
LOG.warning(
"flake8-per-file-ignores plugin is incompatible with "
"flake8>=3.7 (which implements per-file-ignores itself)."
)
continue
self._load_plugin_from_entrypoint(entry_point)
def _load_plugin_from_entrypoint(self, entry_point, local=False):
"""Load a plugin from a setuptools EntryPoint.
:param EntryPoint entry_point:
EntryPoint to load plugin from.
:param bool local:
Is this a repo-local plugin?
"""
name = entry_point.name
self.plugins[name] = Plugin(name, entry_point, local=local)
self.names.append(name)
LOG.debug('Loaded %r for plugin "%s".', self.plugins[name], name)
def map(self, func, *args, **kwargs):
r"""Call ``func`` with the plugin and \*args and \**kwargs after.
This yields the return value from ``func`` for each plugin.
:param collections.Callable func:
Function to call with each plugin. Signature should at least be:
.. code-block:: python
def myfunc(plugin):
pass
Any extra positional or keyword arguments specified with map will
be passed along to this function after the plugin. The plugin
passed is a :class:`~flake8.plugins.manager.Plugin`.
:param args:
Positional arguments to pass to ``func`` after each plugin.
:param kwargs:
Keyword arguments to pass to ``func`` after each plugin.
"""
for name in self.names:
yield func(self.plugins[name], *args, **kwargs)
def versions(self):
# () -> (str, str)
"""Generate the versions of plugins.
:returns:
Tuples of the plugin_name and version
:rtype:
tuple
"""
plugins_seen: Set[str] = set()
for entry_point_name in self.names:
plugin = self.plugins[entry_point_name]
plugin_name = plugin.plugin_name
if plugin.plugin_name in plugins_seen:
continue
plugins_seen.add(plugin_name)
yield (plugin_name, plugin.version)
def version_for(plugin):
# (Plugin) -> Optional[str]
"""Determine the version of a plugin by its module.
:param plugin:
The loaded plugin
:type plugin:
Plugin
:returns:
version string for the module
:rtype:
str
"""
module_name = plugin.plugin.__module__
try:
module = __import__(module_name)
except ImportError:
return None
return getattr(module, "__version__", None)
class PluginTypeManager:
"""Parent class for most of the specific plugin types."""
namespace: str
def __init__(self, local_plugins=None):
"""Initialize the plugin type's manager.
:param list local_plugins:
Plugins from config file instead of entry-points
"""
self.manager = PluginManager(
self.namespace, local_plugins=local_plugins
)
self.plugins_loaded = False
def __contains__(self, name):
"""Check if the entry-point name is in this plugin type manager."""
LOG.debug('Checking for "%s" in plugin type manager.', name)
return name in self.plugins
def __getitem__(self, name):
"""Retrieve a plugin by its name."""
LOG.debug('Retrieving plugin for "%s".', name)
return self.plugins[name]
def get(self, name, default=None):
"""Retrieve the plugin referred to by ``name`` or return the default.
:param str name:
Name of the plugin to retrieve.
:param default:
Default value to return.
:returns:
Plugin object referred to by name, if it exists.
:rtype:
:class:`Plugin`
"""
if name in self:
return self[name]
return default
@property
def names(self):
"""Proxy attribute to underlying manager."""
return self.manager.names
@property
def plugins(self):
"""Proxy attribute to underlying manager."""
return self.manager.plugins
@staticmethod
def _generate_call_function(method_name, optmanager, *args, **kwargs):
def generated_function(plugin):
method = getattr(plugin, method_name, None)
if method is not None and callable(method):
return method(optmanager, *args, **kwargs)
return generated_function
def load_plugins(self):
"""Load all plugins of this type that are managed by this manager."""
if self.plugins_loaded:
return
def load_plugin(plugin):
"""Call each plugin's load_plugin method."""
return plugin.load_plugin()
plugins = list(self.manager.map(load_plugin))
# Do not set plugins_loaded if we run into an exception
self.plugins_loaded = True
return plugins
def register_plugin_versions(self, optmanager):
"""Register the plugins and their versions with the OptionManager."""
self.load_plugins()
for (plugin_name, version) in self.manager.versions():
optmanager.register_plugin(name=plugin_name, version=version)
def register_options(self, optmanager):
"""Register all of the checkers' options to the OptionManager."""
self.load_plugins()
call_register_options = self._generate_call_function(
"register_options", optmanager
)
list(self.manager.map(call_register_options))
def provide_options(self, optmanager, options, extra_args):
"""Provide parsed options and extra arguments to the plugins."""
call_provide_options = self._generate_call_function(
"provide_options", optmanager, options, extra_args
)
list(self.manager.map(call_provide_options))
class Checkers(PluginTypeManager):
"""All of the checkers registered through entry-points or config."""
namespace = "flake8.extension"
def checks_expecting(self, argument_name):
"""Retrieve checks that expect an argument with the specified name.
Find all checker plugins that are expecting a specific argument.
"""
for plugin in self.plugins.values():
if argument_name == plugin.parameter_names[0]:
yield plugin
def to_dictionary(self):
"""Return a dictionary of AST and line-based plugins."""
return {
"ast_plugins": [
plugin.to_dictionary() for plugin in self.ast_plugins
],
"logical_line_plugins": [
plugin.to_dictionary() for plugin in self.logical_line_plugins
],
"physical_line_plugins": [
plugin.to_dictionary() for plugin in self.physical_line_plugins
],
}
def register_options(self, optmanager):
"""Register all of the checkers' options to the OptionManager.
This also ensures that plugins that are not part of a group and are
enabled by default are enabled on the option manager.
"""
# NOTE(sigmavirus24) We reproduce a little of
# PluginTypeManager.register_options to reduce the number of times
# that we loop over the list of plugins. Instead of looping twice,
# option registration and enabling the plugin, we loop once with one
# function to map over the plugins.
self.load_plugins()
call_register_options = self._generate_call_function(
"register_options", optmanager
)
def register_and_enable(plugin):
call_register_options(plugin)
if plugin.group() is None and not plugin.off_by_default:
plugin.enable(optmanager)
list(self.manager.map(register_and_enable))
@property
def ast_plugins(self):
"""List of plugins that expect the AST tree."""
plugins = getattr(self, "_ast_plugins", [])
if not plugins:
plugins = list(self.checks_expecting("tree"))
self._ast_plugins = plugins
return plugins
@property
def logical_line_plugins(self):
"""List of plugins that expect the logical lines."""
plugins = getattr(self, "_logical_line_plugins", [])
if not plugins:
plugins = list(self.checks_expecting("logical_line"))
self._logical_line_plugins = plugins
return plugins
@property
def physical_line_plugins(self):
"""List of plugins that expect the physical lines."""
plugins = getattr(self, "_physical_line_plugins", [])
if not plugins:
plugins = list(self.checks_expecting("physical_line"))
self._physical_line_plugins = plugins
return plugins
class ReportFormatters(PluginTypeManager):
"""All of the report formatters registered through entry-points/config."""
namespace = "flake8.report"

View file

@ -0,0 +1,186 @@
"""Plugin built-in to Flake8 to treat pyflakes as a plugin."""
import os
from typing import List
import pyflakes.checker
from flake8 import utils
FLAKE8_PYFLAKES_CODES = {
"UnusedImport": "F401",
"ImportShadowedByLoopVar": "F402",
"ImportStarUsed": "F403",
"LateFutureImport": "F404",
"ImportStarUsage": "F405",
"ImportStarNotPermitted": "F406",
"FutureFeatureNotDefined": "F407",
"PercentFormatInvalidFormat": "F501",
"PercentFormatExpectedMapping": "F502",
"PercentFormatExpectedSequence": "F503",
"PercentFormatExtraNamedArguments": "F504",
"PercentFormatMissingArgument": "F505",
"PercentFormatMixedPositionalAndNamed": "F506",
"PercentFormatPositionalCountMismatch": "F507",
"PercentFormatStarRequiresSequence": "F508",
"PercentFormatUnsupportedFormatCharacter": "F509",
"StringDotFormatInvalidFormat": "F521",
"StringDotFormatExtraNamedArguments": "F522",
"StringDotFormatExtraPositionalArguments": "F523",
"StringDotFormatMissingArgument": "F524",
"StringDotFormatMixingAutomatic": "F525",
"FStringMissingPlaceholders": "F541",
"MultiValueRepeatedKeyLiteral": "F601",
"MultiValueRepeatedKeyVariable": "F602",
"TooManyExpressionsInStarredAssignment": "F621",
"TwoStarredExpressions": "F622",
"AssertTuple": "F631",
"IsLiteral": "F632",
"InvalidPrintSyntax": "F633",
"IfTuple": "F634",
"BreakOutsideLoop": "F701",
"ContinueOutsideLoop": "F702",
"ContinueInFinally": "F703",
"YieldOutsideFunction": "F704",
"ReturnWithArgsInsideGenerator": "F705",
"ReturnOutsideFunction": "F706",
"DefaultExceptNotLast": "F707",
"DoctestSyntaxError": "F721",
"ForwardAnnotationSyntaxError": "F722",
"CommentAnnotationSyntaxError": "F723",
"RedefinedWhileUnused": "F811",
"RedefinedInListComp": "F812",
"UndefinedName": "F821",
"UndefinedExport": "F822",
"UndefinedLocal": "F823",
"DuplicateArgument": "F831",
"UnusedVariable": "F841",
"RaiseNotImplemented": "F901",
}
class FlakesChecker(pyflakes.checker.Checker):
"""Subclass the Pyflakes checker to conform with the flake8 API."""
name = "pyflakes"
version = pyflakes.__version__
with_doctest = False
include_in_doctest: List[str] = []
exclude_from_doctest: List[str] = []
def __init__(self, tree, file_tokens, filename):
"""Initialize the PyFlakes plugin with an AST tree and filename."""
filename = utils.normalize_path(filename)
with_doctest = self.with_doctest
included_by = [
include
for include in self.include_in_doctest
if include != "" and filename.startswith(include)
]
if included_by:
with_doctest = True
for exclude in self.exclude_from_doctest:
if exclude != "" and filename.startswith(exclude):
with_doctest = False
overlaped_by = [
include
for include in included_by
if include.startswith(exclude)
]
if overlaped_by:
with_doctest = True
super().__init__(
tree,
filename=filename,
withDoctest=with_doctest,
file_tokens=file_tokens,
)
@classmethod
def add_options(cls, parser):
"""Register options for PyFlakes on the Flake8 OptionManager."""
parser.add_option(
"--builtins",
parse_from_config=True,
comma_separated_list=True,
help="define more built-ins, comma separated",
)
parser.add_option(
"--doctests",
default=False,
action="store_true",
parse_from_config=True,
help="also check syntax of the doctests",
)
parser.add_option(
"--include-in-doctest",
default="",
dest="include_in_doctest",
parse_from_config=True,
comma_separated_list=True,
normalize_paths=True,
help="Run doctests only on these files",
)
parser.add_option(
"--exclude-from-doctest",
default="",
dest="exclude_from_doctest",
parse_from_config=True,
comma_separated_list=True,
normalize_paths=True,
help="Skip these files when running doctests",
)
@classmethod
def parse_options(cls, options):
"""Parse option values from Flake8's OptionManager."""
if options.builtins:
cls.builtIns = cls.builtIns.union(options.builtins)
cls.with_doctest = options.doctests
included_files = []
for included_file in options.include_in_doctest:
if included_file == "":
continue
if not included_file.startswith((os.sep, "./", "~/")):
included_files.append(f"./{included_file}")
else:
included_files.append(included_file)
cls.include_in_doctest = utils.normalize_paths(included_files)
excluded_files = []
for excluded_file in options.exclude_from_doctest:
if excluded_file == "":
continue
if not excluded_file.startswith((os.sep, "./", "~/")):
excluded_files.append(f"./{excluded_file}")
else:
excluded_files.append(excluded_file)
cls.exclude_from_doctest = utils.normalize_paths(excluded_files)
inc_exc = set(cls.include_in_doctest).intersection(
cls.exclude_from_doctest
)
if inc_exc:
raise ValueError(
f"{inc_exc!r} was specified in both the "
f"include-in-doctest and exclude-from-doctest "
f"options. You are not allowed to specify it in "
f"both for doctesting."
)
def run(self):
"""Run the plugin."""
for message in self.messages:
col = getattr(message, "col", 0)
yield (
message.lineno,
col,
"{} {}".format(
FLAKE8_PYFLAKES_CODES.get(type(message).__name__, "F999"),
message.message % message.message_args,
),
message.__class__,
)

View file

@ -0,0 +1,469 @@
"""Module containing our file processor that tokenizes a file for checks."""
import argparse
import ast
import contextlib
import logging
import tokenize
from typing import Any
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from typing import Tuple
import flake8
from flake8 import defaults
from flake8 import utils
LOG = logging.getLogger(__name__)
PyCF_ONLY_AST = 1024
NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE])
SKIP_TOKENS = frozenset(
[tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT]
)
_Token = Tuple[int, str, Tuple[int, int], Tuple[int, int], str]
_LogicalMapping = List[Tuple[int, Tuple[int, int]]]
_Logical = Tuple[List[str], List[str], _LogicalMapping]
class FileProcessor:
"""Processes a file and holdes state.
This processes a file by generating tokens, logical and physical lines,
and AST trees. This also provides a way of passing state about the file
to checks expecting that state. Any public attribute on this object can
be requested by a plugin. The known public attributes are:
- :attr:`blank_before`
- :attr:`blank_lines`
- :attr:`checker_state`
- :attr:`indent_char`
- :attr:`indent_level`
- :attr:`line_number`
- :attr:`logical_line`
- :attr:`max_line_length`
- :attr:`max_doc_length`
- :attr:`multiline`
- :attr:`noqa`
- :attr:`previous_indent_level`
- :attr:`previous_logical`
- :attr:`previous_unindented_logical_line`
- :attr:`tokens`
- :attr:`file_tokens`
- :attr:`total_lines`
- :attr:`verbose`
"""
#: always ``False``, included for compatibility
noqa = False
def __init__(
self,
filename: str,
options: argparse.Namespace,
lines: Optional[List[str]] = None,
) -> None:
"""Initialice our file processor.
:param str filename:
Name of the file to process
"""
self.options = options
self.filename = filename
self.lines = lines if lines is not None else self.read_lines()
self.strip_utf_bom()
# Defaults for public attributes
#: Number of preceding blank lines
self.blank_before = 0
#: Number of blank lines
self.blank_lines = 0
#: Checker states for each plugin?
self._checker_states: Dict[str, Dict[Any, Any]] = {}
#: Current checker state
self.checker_state: Dict[Any, Any] = {}
#: User provided option for hang closing
self.hang_closing = options.hang_closing
#: Character used for indentation
self.indent_char: Optional[str] = None
#: Current level of indentation
self.indent_level = 0
#: Number of spaces used for indentation
self.indent_size = options.indent_size
#: String representing the space indentation (DEPRECATED)
self.indent_size_str = str(self.indent_size)
#: Line number in the file
self.line_number = 0
#: Current logical line
self.logical_line = ""
#: Maximum line length as configured by the user
self.max_line_length = options.max_line_length
#: Maximum docstring / comment line length as configured by the user
self.max_doc_length = options.max_doc_length
#: Whether the current physical line is multiline
self.multiline = False
#: Previous level of indentation
self.previous_indent_level = 0
#: Previous logical line
self.previous_logical = ""
#: Previous unindented (i.e. top-level) logical line
self.previous_unindented_logical_line = ""
#: Current set of tokens
self.tokens: List[_Token] = []
#: Total number of lines in the file
self.total_lines = len(self.lines)
#: Verbosity level of Flake8
self.verbose = options.verbose
#: Statistics dictionary
self.statistics = {"logical lines": 0}
self._file_tokens: Optional[List[_Token]] = None
# map from line number to the line we'll search for `noqa` in
self._noqa_line_mapping: Optional[Dict[int, str]] = None
@property
def file_tokens(self) -> List[_Token]:
"""Return the complete set of tokens for a file."""
if self._file_tokens is None:
line_iter = iter(self.lines)
self._file_tokens = list(
tokenize.generate_tokens(lambda: next(line_iter))
)
return self._file_tokens
@contextlib.contextmanager
def inside_multiline(
self, line_number: int
) -> Generator[None, None, None]:
"""Context-manager to toggle the multiline attribute."""
self.line_number = line_number
self.multiline = True
yield
self.multiline = False
def reset_blank_before(self) -> None:
"""Reset the blank_before attribute to zero."""
self.blank_before = 0
def delete_first_token(self) -> None:
"""Delete the first token in the list of tokens."""
del self.tokens[0]
def visited_new_blank_line(self) -> None:
"""Note that we visited a new blank line."""
self.blank_lines += 1
def update_state(self, mapping: _LogicalMapping) -> None:
"""Update the indent level based on the logical line mapping."""
(start_row, start_col) = mapping[0][1]
start_line = self.lines[start_row - 1]
self.indent_level = expand_indent(start_line[:start_col])
if self.blank_before < self.blank_lines:
self.blank_before = self.blank_lines
def update_checker_state_for(self, plugin: Dict[str, Any]) -> None:
"""Update the checker_state attribute for the plugin."""
if "checker_state" in plugin["parameters"]:
self.checker_state = self._checker_states.setdefault(
plugin["name"], {}
)
def next_logical_line(self) -> None:
"""Record the previous logical line.
This also resets the tokens list and the blank_lines count.
"""
if self.logical_line:
self.previous_indent_level = self.indent_level
self.previous_logical = self.logical_line
if not self.indent_level:
self.previous_unindented_logical_line = self.logical_line
self.blank_lines = 0
self.tokens = []
def build_logical_line_tokens(self) -> _Logical:
"""Build the mapping, comments, and logical line lists."""
logical = []
comments = []
mapping: _LogicalMapping = []
length = 0
previous_row = previous_column = None
for token_type, text, start, end, line in self.tokens:
if token_type in SKIP_TOKENS:
continue
if not mapping:
mapping = [(0, start)]
if token_type == tokenize.COMMENT:
comments.append(text)
continue
if token_type == tokenize.STRING:
text = mutate_string(text)
if previous_row:
(start_row, start_column) = start
if previous_row != start_row:
row_index = previous_row - 1
column_index = previous_column - 1
previous_text = self.lines[row_index][column_index]
if previous_text == "," or (
previous_text not in "{[(" and text not in "}])"
):
text = f" {text}"
elif previous_column != start_column:
text = line[previous_column:start_column] + text
logical.append(text)
length += len(text)
mapping.append((length, end))
(previous_row, previous_column) = end
return comments, logical, mapping
def build_ast(self) -> ast.AST:
"""Build an abstract syntax tree from the list of lines."""
return ast.parse("".join(self.lines))
def build_logical_line(self) -> Tuple[str, str, _LogicalMapping]:
"""Build a logical line from the current tokens list."""
comments, logical, mapping_list = self.build_logical_line_tokens()
joined_comments = "".join(comments)
self.logical_line = "".join(logical)
self.statistics["logical lines"] += 1
return joined_comments, self.logical_line, mapping_list
def split_line(self, token: _Token) -> Generator[str, None, None]:
"""Split a physical line's line based on new-lines.
This also auto-increments the line number for the caller.
"""
for line in token[1].split("\n")[:-1]:
yield line
self.line_number += 1
def keyword_arguments_for(
self,
parameters: Dict[str, bool],
arguments: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Generate the keyword arguments for a list of parameters."""
if arguments is None:
arguments = {}
for param, required in parameters.items():
if param in arguments:
continue
try:
arguments[param] = getattr(self, param)
except AttributeError as exc:
if required:
LOG.exception(exc)
raise
else:
LOG.warning(
'Plugin requested optional parameter "%s" '
"but this is not an available parameter.",
param,
)
return arguments
def generate_tokens(self) -> Generator[_Token, None, None]:
"""Tokenize the file and yield the tokens."""
for token in tokenize.generate_tokens(self.next_line):
if token[2][0] > self.total_lines:
break
self.tokens.append(token)
yield token
def _noqa_line_range(self, min_line: int, max_line: int) -> Dict[int, str]:
line_range = range(min_line, max_line + 1)
joined = "".join(self.lines[min_line - 1 : max_line])
return dict.fromkeys(line_range, joined)
def noqa_line_for(self, line_number: int) -> Optional[str]:
"""Retrieve the line which will be used to determine noqa."""
if self._noqa_line_mapping is None:
try:
file_tokens = self.file_tokens
except (tokenize.TokenError, SyntaxError):
# if we failed to parse the file tokens, we'll always fail in
# the future, so set this so the code does not try again
self._noqa_line_mapping = {}
else:
ret = {}
min_line = len(self.lines) + 2
max_line = -1
for tp, _, (s_line, _), (e_line, _), _ in file_tokens:
if tp == tokenize.ENDMARKER:
break
min_line = min(min_line, s_line)
max_line = max(max_line, e_line)
if tp in (tokenize.NL, tokenize.NEWLINE):
ret.update(self._noqa_line_range(min_line, max_line))
min_line = len(self.lines) + 2
max_line = -1
# in newer versions of python, a `NEWLINE` token is inserted
# at the end of the file even if it doesn't have one.
# on old pythons, they will not have hit a `NEWLINE`
if max_line != -1:
ret.update(self._noqa_line_range(min_line, max_line))
self._noqa_line_mapping = ret
# NOTE(sigmavirus24): Some plugins choose to report errors for empty
# files on Line 1. In those cases, we shouldn't bother trying to
# retrieve a physical line (since none exist).
return self._noqa_line_mapping.get(line_number)
def next_line(self) -> str:
"""Get the next line from the list."""
if self.line_number >= self.total_lines:
return ""
line = self.lines[self.line_number]
self.line_number += 1
if self.indent_char is None and line[:1] in defaults.WHITESPACE:
self.indent_char = line[0]
return line
def read_lines(self) -> List[str]:
"""Read the lines for this file checker."""
if self.filename is None or self.filename == "-":
self.filename = self.options.stdin_display_name or "stdin"
lines = self.read_lines_from_stdin()
else:
lines = self.read_lines_from_filename()
return lines
def read_lines_from_filename(self) -> List[str]:
"""Read the lines for a file."""
try:
with tokenize.open(self.filename) as fd:
return fd.readlines()
except (SyntaxError, UnicodeError):
# If we can't detect the codec with tokenize.detect_encoding, or
# the detected encoding is incorrect, just fallback to latin-1.
with open(self.filename, encoding="latin-1") as fd:
return fd.readlines()
def read_lines_from_stdin(self) -> List[str]:
"""Read the lines from standard in."""
return utils.stdin_get_lines()
def should_ignore_file(self) -> bool:
"""Check if ``flake8: noqa`` is in the file to be ignored.
:returns:
True if a line matches :attr:`defaults.NOQA_FILE`,
otherwise False
:rtype:
bool
"""
if not self.options.disable_noqa and any(
defaults.NOQA_FILE.match(line) for line in self.lines
):
return True
elif any(defaults.NOQA_FILE.search(line) for line in self.lines):
LOG.warning(
"Detected `flake8: noqa` on line with code. To ignore an "
"error on a line use `noqa` instead."
)
return False
else:
return False
def strip_utf_bom(self) -> None:
"""Strip the UTF bom from the lines of the file."""
if not self.lines:
# If we have nothing to analyze quit early
return
first_byte = ord(self.lines[0][0])
if first_byte not in (0xEF, 0xFEFF):
return
# If the first byte of the file is a UTF-8 BOM, strip it
if first_byte == 0xFEFF:
self.lines[0] = self.lines[0][1:]
elif self.lines[0][:3] == "\xEF\xBB\xBF":
self.lines[0] = self.lines[0][3:]
def is_eol_token(token: _Token) -> bool:
"""Check if the token is an end-of-line token."""
return token[0] in NEWLINE or token[4][token[3][1] :].lstrip() == "\\\n"
def is_multiline_string(token: _Token) -> bool:
"""Check if this is a multiline string."""
return token[0] == tokenize.STRING and "\n" in token[1]
def token_is_newline(token: _Token) -> bool:
"""Check if the token type is a newline token type."""
return token[0] in NEWLINE
def count_parentheses(current_parentheses_count: int, token_text: str) -> int:
"""Count the number of parentheses."""
if token_text in "([{": # nosec
return current_parentheses_count + 1
elif token_text in "}])": # nosec
return current_parentheses_count - 1
return current_parentheses_count
def log_token(log: logging.Logger, token: _Token) -> None:
"""Log a token to a provided logging object."""
if token[2][0] == token[3][0]:
pos = "[{}:{}]".format(token[2][1] or "", token[3][1])
else:
pos = f"l.{token[3][0]}"
log.log(
flake8._EXTRA_VERBOSE,
"l.%s\t%s\t%s\t%r"
% (token[2][0], pos, tokenize.tok_name[token[0]], token[1]),
)
def expand_indent(line: str) -> int:
r"""Return the amount of indentation.
Tabs are expanded to the next multiple of 8.
>>> expand_indent(' ')
4
>>> expand_indent('\t')
8
>>> expand_indent(' \t')
8
>>> expand_indent(' \t')
16
"""
return len(line.expandtabs(8))
# NOTE(sigmavirus24): This was taken wholesale from
# https://github.com/PyCQA/pycodestyle. The in-line comments were edited to be
# more descriptive.
def mutate_string(text: str) -> str:
"""Replace contents with 'xxx' to prevent syntax matching.
>>> mutate_string('"abc"')
'"xxx"'
>>> mutate_string("'''abc'''")
"'''xxx'''"
>>> mutate_string("r'abc'")
"r'xxx'"
"""
# NOTE(sigmavirus24): If there are string modifiers (e.g., b, u, r)
# use the last "character" to determine if we're using single or double
# quotes and then find the first instance of it
start = text.index(text[-1]) + 1
end = len(text) - 1
# Check for triple-quoted strings
if text[-3:] in ('"""', "'''"):
start += 2
end -= 2
return text[:start] + "x" * (end - start) + text[end:]

View file

@ -0,0 +1,139 @@
"""Statistic collection logic for Flake8."""
import collections
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from flake8.style_guide import Violation
class Statistics:
"""Manager of aggregated statistics for a run of Flake8."""
def __init__(self) -> None:
"""Initialize the underlying dictionary for our statistics."""
self._store: Dict[Key, "Statistic"] = {}
def error_codes(self) -> List[str]:
"""Return all unique error codes stored.
:returns:
Sorted list of error codes.
:rtype:
list(str)
"""
return sorted({key.code for key in self._store})
def record(self, error: "Violation") -> None:
"""Add the fact that the error was seen in the file.
:param error:
The Violation instance containing the information about the
violation.
:type error:
flake8.style_guide.Violation
"""
key = Key.create_from(error)
if key not in self._store:
self._store[key] = Statistic.create_from(error)
self._store[key].increment()
def statistics_for(
self, prefix: str, filename: Optional[str] = None
) -> Generator["Statistic", None, None]:
"""Generate statistics for the prefix and filename.
If you have a :class:`Statistics` object that has recorded errors,
you can generate the statistics for a prefix (e.g., ``E``, ``E1``,
``W50``, ``W503``) with the optional filter of a filename as well.
.. code-block:: python
>>> stats = Statistics()
>>> stats.statistics_for('E12',
filename='src/flake8/statistics.py')
<generator ...>
>>> stats.statistics_for('W')
<generator ...>
:param str prefix:
The error class or specific error code to find statistics for.
:param str filename:
(Optional) The filename to further filter results by.
:returns:
Generator of instances of :class:`Statistic`
"""
matching_errors = sorted(
key for key in self._store if key.matches(prefix, filename)
)
for error_code in matching_errors:
yield self._store[error_code]
class Key(collections.namedtuple("Key", ["filename", "code"])):
"""Simple key structure for the Statistics dictionary.
To make things clearer, easier to read, and more understandable, we use a
namedtuple here for all Keys in the underlying dictionary for the
Statistics object.
"""
__slots__ = ()
@classmethod
def create_from(cls, error: "Violation") -> "Key":
"""Create a Key from :class:`flake8.style_guide.Violation`."""
return cls(filename=error.filename, code=error.code)
def matches(self, prefix: str, filename: Optional[str]) -> bool:
"""Determine if this key matches some constraints.
:param str prefix:
The error code prefix that this key's error code should start with.
:param str filename:
The filename that we potentially want to match on. This can be
None to only match on error prefix.
:returns:
True if the Key's code starts with the prefix and either filename
is None, or the Key's filename matches the value passed in.
:rtype:
bool
"""
return self.code.startswith(prefix) and (
filename is None or self.filename == filename
)
class Statistic:
"""Simple wrapper around the logic of each statistic.
Instead of maintaining a simple but potentially hard to reason about
tuple, we create a namedtuple which has attributes and a couple
convenience methods on it.
"""
def __init__(
self, error_code: str, filename: str, message: str, count: int
) -> None:
"""Initialize our Statistic."""
self.error_code = error_code
self.filename = filename
self.message = message
self.count = count
@classmethod
def create_from(cls, error: "Violation") -> "Statistic":
"""Create a Statistic from a :class:`flake8.style_guide.Violation`."""
return cls(
error_code=error.code,
filename=error.filename,
message=error.text,
count=0,
)
def increment(self) -> None:
"""Increment the number of times we've seen this error in this file."""
self.count += 1

View file

@ -0,0 +1,613 @@
"""Implementation of the StyleGuide used by Flake8."""
import argparse
import collections
import contextlib
import copy
import enum
import functools
import itertools
import linecache
import logging
from typing import Dict
from typing import Generator
from typing import List
from typing import Match
from typing import Optional
from typing import Sequence
from typing import Set
from typing import Tuple
from typing import Union
from flake8 import defaults
from flake8 import statistics
from flake8 import utils
from flake8.formatting import base as base_formatter
__all__ = ("StyleGuide",)
LOG = logging.getLogger(__name__)
class Selected(enum.Enum):
"""Enum representing an explicitly or implicitly selected code."""
Explicitly = "explicitly selected"
Implicitly = "implicitly selected"
class Ignored(enum.Enum):
"""Enum representing an explicitly or implicitly ignored code."""
Explicitly = "explicitly ignored"
Implicitly = "implicitly ignored"
class Decision(enum.Enum):
"""Enum representing whether a code should be ignored or selected."""
Ignored = "ignored error"
Selected = "selected error"
@functools.lru_cache(maxsize=512)
def find_noqa(physical_line: str) -> Optional[Match[str]]:
return defaults.NOQA_INLINE_REGEXP.search(physical_line)
class Violation(
collections.namedtuple(
"Violation",
[
"code",
"filename",
"line_number",
"column_number",
"text",
"physical_line",
],
)
):
"""Class representing a violation reported by Flake8."""
def is_inline_ignored(self, disable_noqa: bool) -> bool:
"""Determine if a comment has been added to ignore this line.
:param bool disable_noqa:
Whether or not users have provided ``--disable-noqa``.
:returns:
True if error is ignored in-line, False otherwise.
:rtype:
bool
"""
physical_line = self.physical_line
# TODO(sigmavirus24): Determine how to handle stdin with linecache
if disable_noqa:
return False
if physical_line is None:
physical_line = linecache.getline(self.filename, self.line_number)
noqa_match = find_noqa(physical_line)
if noqa_match is None:
LOG.debug("%r is not inline ignored", self)
return False
codes_str = noqa_match.groupdict()["codes"]
if codes_str is None:
LOG.debug("%r is ignored by a blanket ``# noqa``", self)
return True
codes = set(utils.parse_comma_separated_list(codes_str))
if self.code in codes or self.code.startswith(tuple(codes)):
LOG.debug(
"%r is ignored specifically inline with ``# noqa: %s``",
self,
codes_str,
)
return True
LOG.debug(
"%r is not ignored inline with ``# noqa: %s``", self, codes_str
)
return False
def is_in(self, diff: Dict[str, Set[int]]) -> bool:
"""Determine if the violation is included in a diff's line ranges.
This function relies on the parsed data added via
:meth:`~StyleGuide.add_diff_ranges`. If that has not been called and
we are not evaluating files in a diff, then this will always return
True. If there are diff ranges, then this will return True if the
line number in the error falls inside one of the ranges for the file
(and assuming the file is part of the diff data). If there are diff
ranges, this will return False if the file is not part of the diff
data or the line number of the error is not in any of the ranges of
the diff.
:returns:
True if there is no diff or if the error is in the diff's line
number ranges. False if the error's line number falls outside
the diff's line number ranges.
:rtype:
bool
"""
if not diff:
return True
# NOTE(sigmavirus24): The parsed diff will be a defaultdict with
# a set as the default value (if we have received it from
# flake8.utils.parse_unified_diff). In that case ranges below
# could be an empty set (which is False-y) or if someone else
# is using this API, it could be None. If we could guarantee one
# or the other, we would check for it more explicitly.
line_numbers = diff.get(self.filename)
if not line_numbers:
return False
return self.line_number in line_numbers
class DecisionEngine:
"""A class for managing the decision process around violations.
This contains the logic for whether a violation should be reported or
ignored.
"""
def __init__(self, options: argparse.Namespace) -> None:
"""Initialize the engine."""
self.cache: Dict[str, Decision] = {}
self.selected = tuple(options.select)
self.extended_selected = tuple(
sorted(options.extended_default_select, reverse=True)
)
self.enabled_extensions = tuple(options.enable_extensions)
self.all_selected = tuple(
sorted(
itertools.chain(
self.selected,
options.extend_select,
self.enabled_extensions,
),
reverse=True,
)
)
self.ignored = tuple(
sorted(
itertools.chain(options.ignore, options.extend_ignore),
reverse=True,
)
)
self.using_default_ignore = set(self.ignored) == set(
defaults.IGNORE
).union(options.extended_default_ignore)
self.using_default_select = set(self.selected) == set(defaults.SELECT)
def _in_all_selected(self, code: str) -> bool:
return bool(self.all_selected) and code.startswith(self.all_selected)
def _in_extended_selected(self, code: str) -> bool:
return bool(self.extended_selected) and code.startswith(
self.extended_selected
)
def was_selected(self, code: str) -> Union[Selected, Ignored]:
"""Determine if the code has been selected by the user.
:param str code:
The code for the check that has been run.
:returns:
Selected.Implicitly if the selected list is empty,
Selected.Explicitly if the selected list is not empty and a match
was found,
Ignored.Implicitly if the selected list is not empty but no match
was found.
"""
if self._in_all_selected(code):
return Selected.Explicitly
if not self.all_selected and self._in_extended_selected(code):
# If it was not explicitly selected, it may have been implicitly
# selected because the check comes from a plugin that is enabled by
# default
return Selected.Implicitly
return Ignored.Implicitly
def was_ignored(self, code: str) -> Union[Selected, Ignored]:
"""Determine if the code has been ignored by the user.
:param str code:
The code for the check that has been run.
:returns:
Selected.Implicitly if the ignored list is empty,
Ignored.Explicitly if the ignored list is not empty and a match was
found,
Selected.Implicitly if the ignored list is not empty but no match
was found.
"""
if self.ignored and code.startswith(self.ignored):
return Ignored.Explicitly
return Selected.Implicitly
def more_specific_decision_for(self, code: str) -> Decision:
select = find_first_match(code, self.all_selected)
extra_select = find_first_match(code, self.extended_selected)
ignore = find_first_match(code, self.ignored)
if select and ignore:
# If the violation code appears in both the select and ignore
# lists (in some fashion) then if we're using the default ignore
# list and a custom select list we should select the code. An
# example usage looks like this:
# A user has a code that would generate an E126 violation which
# is in our default ignore list and they specify select=E.
# We should be reporting that violation. This logic changes,
# however, if they specify select and ignore such that both match.
# In that case we fall through to our find_more_specific call.
# If, however, the user hasn't specified a custom select, and
# we're using the defaults for both select and ignore then the
# more specific rule must win. In most cases, that will be to
# ignore the violation since our default select list is very
# high-level and our ignore list is highly specific.
if self.using_default_ignore and not self.using_default_select:
return Decision.Selected
return find_more_specific(select, ignore)
if extra_select and ignore:
# At this point, select is false-y. Now we need to check if the
# code is in our extended select list and our ignore list. This is
# a *rare* case as we see little usage of the extended select list
# that plugins can use, so I suspect this section may change to
# look a little like the block above in which we check if we're
# using our default ignore list.
return find_more_specific(extra_select, ignore)
if select or (extra_select and self.using_default_select):
# Here, ignore was false-y and the user has either selected
# explicitly the violation or the violation is covered by
# something in the extended select list and we're using the
# default select list. In either case, we want the violation to be
# selected.
return Decision.Selected
if select is None and (
extra_select is None or not self.using_default_ignore
):
return Decision.Ignored
if (select is None and not self.using_default_select) and (
ignore is None and self.using_default_ignore
):
return Decision.Ignored
return Decision.Selected
def make_decision(self, code: str) -> Decision:
"""Decide if code should be ignored or selected."""
LOG.debug('Deciding if "%s" should be reported', code)
selected = self.was_selected(code)
ignored = self.was_ignored(code)
LOG.debug(
'The user configured "%s" to be "%s", "%s"',
code,
selected,
ignored,
)
if (
selected is Selected.Explicitly or selected is Selected.Implicitly
) and ignored is Selected.Implicitly:
decision = Decision.Selected
elif (
selected is Selected.Explicitly and ignored is Ignored.Explicitly
) or (
selected is Ignored.Implicitly and ignored is Selected.Implicitly
):
decision = self.more_specific_decision_for(code)
elif selected is Ignored.Implicitly or ignored is Ignored.Explicitly:
decision = Decision.Ignored # pylint: disable=R0204
return decision
def decision_for(self, code: str) -> Decision:
"""Return the decision for a specific code.
This method caches the decisions for codes to avoid retracing the same
logic over and over again. We only care about the select and ignore
rules as specified by the user in their configuration files and
command-line flags.
This method does not look at whether the specific line is being
ignored in the file itself.
:param str code:
The code for the check that has been run.
"""
decision = self.cache.get(code)
if decision is None:
decision = self.make_decision(code)
self.cache[code] = decision
LOG.debug('"%s" will be "%s"', code, decision)
return decision
class StyleGuideManager:
"""Manage multiple style guides for a single run."""
def __init__(
self,
options: argparse.Namespace,
formatter: base_formatter.BaseFormatter,
decider: Optional[DecisionEngine] = None,
) -> None:
"""Initialize our StyleGuide.
.. todo:: Add parameter documentation.
"""
self.options = options
self.formatter = formatter
self.stats = statistics.Statistics()
self.decider = decider or DecisionEngine(options)
self.style_guides: List[StyleGuide] = []
self.default_style_guide = StyleGuide(
options, formatter, self.stats, decider=decider
)
self.style_guides = list(
itertools.chain(
[self.default_style_guide],
self.populate_style_guides_with(options),
)
)
def populate_style_guides_with(
self, options: argparse.Namespace
) -> Generator["StyleGuide", None, None]:
"""Generate style guides from the per-file-ignores option.
:param options:
The original options parsed from the CLI and config file.
:type options:
:class:`~argparse.Namespace`
:returns:
A copy of the default style guide with overridden values.
:rtype:
:class:`~flake8.style_guide.StyleGuide`
"""
per_file = utils.parse_files_to_codes_mapping(options.per_file_ignores)
for filename, violations in per_file:
yield self.default_style_guide.copy(
filename=filename, extend_ignore_with=violations
)
@functools.lru_cache(maxsize=None)
def style_guide_for(self, filename: str) -> "StyleGuide":
"""Find the StyleGuide for the filename in particular."""
guides = sorted(
(g for g in self.style_guides if g.applies_to(filename)),
key=lambda g: len(g.filename or ""),
)
if len(guides) > 1:
return guides[-1]
return guides[0]
@contextlib.contextmanager
def processing_file(
self, filename: str
) -> Generator["StyleGuide", None, None]:
"""Record the fact that we're processing the file's results."""
guide = self.style_guide_for(filename)
with guide.processing_file(filename):
yield guide
def handle_error(
self,
code: str,
filename: str,
line_number: int,
column_number: Optional[int],
text: str,
physical_line: Optional[str] = None,
) -> int:
"""Handle an error reported by a check.
:param str code:
The error code found, e.g., E123.
:param str filename:
The file in which the error was found.
:param int line_number:
The line number (where counting starts at 1) at which the error
occurs.
:param int column_number:
The column number (where counting starts at 1) at which the error
occurs.
:param str text:
The text of the error message.
:param str physical_line:
The actual physical line causing the error.
:returns:
1 if the error was reported. 0 if it was ignored. This is to allow
for counting of the number of errors found that were not ignored.
:rtype:
int
"""
guide = self.style_guide_for(filename)
return guide.handle_error(
code, filename, line_number, column_number, text, physical_line
)
def add_diff_ranges(self, diffinfo: Dict[str, Set[int]]) -> None:
"""Update the StyleGuides to filter out information not in the diff.
This provides information to the underlying StyleGuides so that only
the errors in the line number ranges are reported.
:param dict diffinfo:
Dictionary mapping filenames to sets of line number ranges.
"""
for guide in self.style_guides:
guide.add_diff_ranges(diffinfo)
class StyleGuide:
"""Manage a Flake8 user's style guide."""
def __init__(
self,
options: argparse.Namespace,
formatter: base_formatter.BaseFormatter,
stats: statistics.Statistics,
filename: Optional[str] = None,
decider: Optional[DecisionEngine] = None,
):
"""Initialize our StyleGuide.
.. todo:: Add parameter documentation.
"""
self.options = options
self.formatter = formatter
self.stats = stats
self.decider = decider or DecisionEngine(options)
self.filename = filename
if self.filename:
self.filename = utils.normalize_path(self.filename)
self._parsed_diff: Dict[str, Set[int]] = {}
def __repr__(self) -> str:
"""Make it easier to debug which StyleGuide we're using."""
return f"<StyleGuide [{self.filename}]>"
def copy(
self,
filename: Optional[str] = None,
extend_ignore_with: Optional[Sequence[str]] = None,
) -> "StyleGuide":
"""Create a copy of this style guide with different values."""
filename = filename or self.filename
options = copy.deepcopy(self.options)
options.ignore.extend(extend_ignore_with or [])
return StyleGuide(
options, self.formatter, self.stats, filename=filename
)
@contextlib.contextmanager
def processing_file(
self, filename: str
) -> Generator["StyleGuide", None, None]:
"""Record the fact that we're processing the file's results."""
self.formatter.beginning(filename)
yield self
self.formatter.finished(filename)
def applies_to(self, filename: str) -> bool:
"""Check if this StyleGuide applies to the file.
:param str filename:
The name of the file with violations that we're potentially
applying this StyleGuide to.
:returns:
True if this applies, False otherwise
:rtype:
bool
"""
if self.filename is None:
return True
return utils.matches_filename(
filename,
patterns=[self.filename],
log_message=f'{self!r} does %(whether)smatch "%(path)s"',
logger=LOG,
)
def should_report_error(self, code: str) -> Decision:
"""Determine if the error code should be reported or ignored.
This method only cares about the select and ignore rules as specified
by the user in their configuration files and command-line flags.
This method does not look at whether the specific line is being
ignored in the file itself.
:param str code:
The code for the check that has been run.
"""
return self.decider.decision_for(code)
def handle_error(
self,
code: str,
filename: str,
line_number: int,
column_number: Optional[int],
text: str,
physical_line: Optional[str] = None,
) -> int:
"""Handle an error reported by a check.
:param str code:
The error code found, e.g., E123.
:param str filename:
The file in which the error was found.
:param int line_number:
The line number (where counting starts at 1) at which the error
occurs.
:param int column_number:
The column number (where counting starts at 1) at which the error
occurs.
:param str text:
The text of the error message.
:param str physical_line:
The actual physical line causing the error.
:returns:
1 if the error was reported. 0 if it was ignored. This is to allow
for counting of the number of errors found that were not ignored.
:rtype:
int
"""
disable_noqa = self.options.disable_noqa
# NOTE(sigmavirus24): Apparently we're provided with 0-indexed column
# numbers so we have to offset that here. Also, if a SyntaxError is
# caught, column_number may be None.
if not column_number:
column_number = 0
error = Violation(
code,
filename,
line_number,
column_number + 1,
text,
physical_line,
)
error_is_selected = (
self.should_report_error(error.code) is Decision.Selected
)
is_not_inline_ignored = error.is_inline_ignored(disable_noqa) is False
is_included_in_diff = error.is_in(self._parsed_diff)
if error_is_selected and is_not_inline_ignored and is_included_in_diff:
self.formatter.handle(error)
self.stats.record(error)
return 1
return 0
def add_diff_ranges(self, diffinfo: Dict[str, Set[int]]) -> None:
"""Update the StyleGuide to filter out information not in the diff.
This provides information to the StyleGuide so that only the errors
in the line number ranges are reported.
:param dict diffinfo:
Dictionary mapping filenames to sets of line number ranges.
"""
self._parsed_diff = diffinfo
def find_more_specific(selected: str, ignored: str) -> Decision:
if selected.startswith(ignored) and selected != ignored:
return Decision.Selected
return Decision.Ignored
def find_first_match(
error_code: str, code_list: Tuple[str, ...]
) -> Optional[str]:
startswith = error_code.startswith
for code in code_list:
if startswith(code):
break
else:
return None
return code

View file

@ -0,0 +1,455 @@
"""Utility methods for flake8."""
import collections
import fnmatch as _fnmatch
import functools
import inspect
import io
import logging
import os
import platform
import re
import sys
import textwrap
import tokenize
from typing import Callable
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from typing import Pattern
from typing import Sequence
from typing import Set
from typing import Tuple
from typing import TYPE_CHECKING
from typing import Union
from flake8 import exceptions
if TYPE_CHECKING:
from flake8.plugins.manager import Plugin
DIFF_HUNK_REGEXP = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$")
COMMA_SEPARATED_LIST_RE = re.compile(r"[,\s]")
LOCAL_PLUGIN_LIST_RE = re.compile(r"[,\t\n\r\f\v]")
def parse_comma_separated_list(
value: str, regexp: Pattern[str] = COMMA_SEPARATED_LIST_RE
) -> List[str]:
"""Parse a comma-separated list.
:param value:
String to be parsed and normalized.
:param regexp:
Compiled regular expression used to split the value when it is a
string.
:type regexp:
_sre.SRE_Pattern
:returns:
List of values with whitespace stripped.
:rtype:
list
"""
assert isinstance(value, str), value
separated = regexp.split(value)
item_gen = (item.strip() for item in separated)
return [item for item in item_gen if item]
_Token = collections.namedtuple("_Token", ("tp", "src"))
_CODE, _FILE, _COLON, _COMMA, _WS = "code", "file", "colon", "comma", "ws"
_EOF = "eof"
_FILE_LIST_TOKEN_TYPES = [
(re.compile(r"[A-Z]+[0-9]*(?=$|\s|,)"), _CODE),
(re.compile(r"[^\s:,]+"), _FILE),
(re.compile(r"\s*:\s*"), _COLON),
(re.compile(r"\s*,\s*"), _COMMA),
(re.compile(r"\s+"), _WS),
]
def _tokenize_files_to_codes_mapping(value: str) -> List[_Token]:
tokens = []
i = 0
while i < len(value):
for token_re, token_name in _FILE_LIST_TOKEN_TYPES:
match = token_re.match(value, i)
if match:
tokens.append(_Token(token_name, match.group().strip()))
i = match.end()
break
else:
raise AssertionError("unreachable", value, i)
tokens.append(_Token(_EOF, ""))
return tokens
def parse_files_to_codes_mapping( # noqa: C901
value_: Union[Sequence[str], str]
) -> List[Tuple[str, List[str]]]:
"""Parse a files-to-codes mapping.
A files-to-codes mapping a sequence of values specified as
`filenames list:codes list ...`. Each of the lists may be separated by
either comma or whitespace tokens.
:param value: String to be parsed and normalized.
:type value: str
"""
if not isinstance(value_, str):
value = "\n".join(value_)
else:
value = value_
ret: List[Tuple[str, List[str]]] = []
if not value.strip():
return ret
class State:
seen_sep = True
seen_colon = False
filenames: List[str] = []
codes: List[str] = []
def _reset() -> None:
if State.codes:
for filename in State.filenames:
ret.append((filename, State.codes))
State.seen_sep = True
State.seen_colon = False
State.filenames = []
State.codes = []
def _unexpected_token() -> exceptions.ExecutionError:
return exceptions.ExecutionError(
f"Expected `per-file-ignores` to be a mapping from file exclude "
f"patterns to ignore codes.\n\n"
f"Configured `per-file-ignores` setting:\n\n"
f"{textwrap.indent(value.strip(), ' ')}"
)
for token in _tokenize_files_to_codes_mapping(value):
# legal in any state: separator sets the sep bit
if token.tp in {_COMMA, _WS}:
State.seen_sep = True
# looking for filenames
elif not State.seen_colon:
if token.tp == _COLON:
State.seen_colon = True
State.seen_sep = True
elif State.seen_sep and token.tp == _FILE:
State.filenames.append(token.src)
State.seen_sep = False
else:
raise _unexpected_token()
# looking for codes
else:
if token.tp == _EOF:
_reset()
elif State.seen_sep and token.tp == _CODE:
State.codes.append(token.src)
State.seen_sep = False
elif State.seen_sep and token.tp == _FILE:
_reset()
State.filenames.append(token.src)
State.seen_sep = False
else:
raise _unexpected_token()
return ret
def normalize_paths(
paths: Sequence[str], parent: str = os.curdir
) -> List[str]:
"""Normalize a list of paths relative to a parent directory.
:returns:
The normalized paths.
:rtype:
[str]
"""
assert isinstance(paths, list), paths
return [normalize_path(p, parent) for p in paths]
def normalize_path(path: str, parent: str = os.curdir) -> str:
"""Normalize a single-path.
:returns:
The normalized path.
:rtype:
str
"""
# NOTE(sigmavirus24): Using os.path.sep and os.path.altsep allow for
# Windows compatibility with both Windows-style paths (c:\\foo\bar) and
# Unix style paths (/foo/bar).
separator = os.path.sep
# NOTE(sigmavirus24): os.path.altsep may be None
alternate_separator = os.path.altsep or ""
if separator in path or (
alternate_separator and alternate_separator in path
):
path = os.path.abspath(os.path.join(parent, path))
return path.rstrip(separator + alternate_separator)
@functools.lru_cache(maxsize=1)
def stdin_get_value() -> str:
"""Get and cache it so plugins can use it."""
stdin_value = sys.stdin.buffer.read()
fd = io.BytesIO(stdin_value)
try:
coding, _ = tokenize.detect_encoding(fd.readline)
fd.seek(0)
return io.TextIOWrapper(fd, coding).read()
except (LookupError, SyntaxError, UnicodeError):
return stdin_value.decode("utf-8")
def stdin_get_lines() -> List[str]:
"""Return lines of stdin split according to file splitting."""
return list(io.StringIO(stdin_get_value()))
def parse_unified_diff(diff: Optional[str] = None) -> Dict[str, Set[int]]:
"""Parse the unified diff passed on stdin.
:returns:
dictionary mapping file names to sets of line numbers
:rtype:
dict
"""
# Allow us to not have to patch out stdin_get_value
if diff is None:
diff = stdin_get_value()
number_of_rows = None
current_path = None
parsed_paths: Dict[str, Set[int]] = collections.defaultdict(set)
for line in diff.splitlines():
if number_of_rows:
if not line or line[0] != "-":
number_of_rows -= 1
# We're in the part of the diff that has lines starting with +, -,
# and ' ' to show context and the changes made. We skip these
# because the information we care about is the filename and the
# range within it.
# When number_of_rows reaches 0, we will once again start
# searching for filenames and ranges.
continue
# NOTE(sigmavirus24): Diffs that we support look roughly like:
# diff a/file.py b/file.py
# ...
# --- a/file.py
# +++ b/file.py
# Below we're looking for that last line. Every diff tool that
# gives us this output may have additional information after
# ``b/file.py`` which it will separate with a \t, e.g.,
# +++ b/file.py\t100644
# Which is an example that has the new file permissions/mode.
# In this case we only care about the file name.
if line[:3] == "+++":
current_path = line[4:].split("\t", 1)[0]
# NOTE(sigmavirus24): This check is for diff output from git.
if current_path[:2] == "b/":
current_path = current_path[2:]
# We don't need to do anything else. We have set up our local
# ``current_path`` variable. We can skip the rest of this loop.
# The next line we will see will give us the hung information
# which is in the next section of logic.
continue
hunk_match = DIFF_HUNK_REGEXP.match(line)
# NOTE(sigmavirus24): pep8/pycodestyle check for:
# line[:3] == '@@ '
# But the DIFF_HUNK_REGEXP enforces that the line start with that
# So we can more simply check for a match instead of slicing and
# comparing.
if hunk_match:
(row, number_of_rows) = (
1 if not group else int(group) for group in hunk_match.groups()
)
assert current_path is not None
parsed_paths[current_path].update(range(row, row + number_of_rows))
# We have now parsed our diff into a dictionary that looks like:
# {'file.py': set(range(10, 16), range(18, 20)), ...}
return parsed_paths
def is_windows() -> bool:
"""Determine if we're running on Windows.
:returns:
True if running on Windows, otherwise False
:rtype:
bool
"""
return os.name == "nt"
def is_using_stdin(paths: List[str]) -> bool:
"""Determine if we're going to read from stdin.
:param list paths:
The paths that we're going to check.
:returns:
True if stdin (-) is in the path, otherwise False
:rtype:
bool
"""
return "-" in paths
def _default_predicate(*args: str) -> bool:
return False
def filenames_from(
arg: str, predicate: Optional[Callable[[str], bool]] = None
) -> Generator[str, None, None]:
"""Generate filenames from an argument.
:param str arg:
Parameter from the command-line.
:param callable predicate:
Predicate to use to filter out filenames. If the predicate
returns ``True`` we will exclude the filename, otherwise we
will yield it. By default, we include every filename
generated.
:returns:
Generator of paths
"""
if predicate is None:
predicate = _default_predicate
if predicate(arg):
return
if os.path.isdir(arg):
for root, sub_directories, files in os.walk(arg):
if predicate(root):
sub_directories[:] = []
continue
# NOTE(sigmavirus24): os.walk() will skip a directory if you
# remove it from the list of sub-directories.
for directory in sub_directories:
joined = os.path.join(root, directory)
if predicate(joined):
sub_directories.remove(directory)
for filename in files:
joined = os.path.join(root, filename)
if not predicate(joined):
yield joined
else:
yield arg
def fnmatch(filename: str, patterns: Sequence[str]) -> bool:
"""Wrap :func:`fnmatch.fnmatch` to add some functionality.
:param str filename:
Name of the file we're trying to match.
:param list patterns:
Patterns we're using to try to match the filename.
:param bool default:
The default value if patterns is empty
:returns:
True if a pattern matches the filename, False if it doesn't.
``default`` if patterns is empty.
"""
if not patterns:
return True
return any(_fnmatch.fnmatch(filename, pattern) for pattern in patterns)
def parameters_for(plugin: "Plugin") -> Dict[str, bool]:
"""Return the parameters for the plugin.
This will inspect the plugin and return either the function parameters
if the plugin is a function or the parameters for ``__init__`` after
``self`` if the plugin is a class.
:param plugin:
The internal plugin object.
:type plugin:
flake8.plugins.manager.Plugin
:returns:
A dictionary mapping the parameter name to whether or not it is
required (a.k.a., is positional only/does not have a default).
:rtype:
dict([(str, bool)])
"""
func = plugin.plugin
is_class = not inspect.isfunction(func)
if is_class: # The plugin is a class
func = plugin.plugin.__init__
parameters = {
parameter.name: parameter.default is parameter.empty
for parameter in inspect.signature(func).parameters.values()
if parameter.kind == parameter.POSITIONAL_OR_KEYWORD
}
if is_class:
parameters.pop("self", None)
return parameters
def matches_filename(
path: str,
patterns: Sequence[str],
log_message: str,
logger: logging.Logger,
) -> bool:
"""Use fnmatch to discern if a path exists in patterns.
:param str path:
The path to the file under question
:param patterns:
The patterns to match the path against.
:type patterns:
list[str]
:param str log_message:
The message used for logging purposes.
:returns:
True if path matches patterns, False otherwise
:rtype:
bool
"""
if not patterns:
return False
basename = os.path.basename(path)
if basename not in {".", ".."} and fnmatch(basename, patterns):
logger.debug(log_message, {"path": basename, "whether": ""})
return True
absolute_path = os.path.abspath(path)
match = fnmatch(absolute_path, patterns)
logger.debug(
log_message,
{"path": absolute_path, "whether": "" if match else "not "},
)
return match
def get_python_version() -> str:
"""Find and format the python implementation and version.
:returns:
Implementation name, version, and platform as a string.
:rtype:
str
"""
return "{} {} on {}".format(
platform.python_implementation(),
platform.python_version(),
platform.system(),
)