This commit is contained in:
Waylon Walker 2022-03-31 20:20:07 -05:00
commit 38355d2442
No known key found for this signature in database
GPG key ID: 66E2BF2B4190EFE4
9083 changed files with 1225834 additions and 0 deletions

View file

@ -0,0 +1 @@
"""Module containing the logic for the Flake8 entry-points."""

View file

@ -0,0 +1,388 @@
"""Module containing the application logic for Flake8."""
import argparse
import logging
import sys
import time
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Type
from typing import TYPE_CHECKING
import flake8
from flake8 import checker
from flake8 import defaults
from flake8 import exceptions
from flake8 import style_guide
from flake8 import utils
from flake8.main import options
from flake8.options import aggregator
from flake8.options import config
from flake8.options import manager
from flake8.plugins import manager as plugin_manager
if TYPE_CHECKING:
from flake8.formatting.base import BaseFormatter
LOG = logging.getLogger(__name__)
class Application:
"""Abstract our application into a class."""
def __init__(self, program="flake8", version=flake8.__version__):
"""Initialize our application.
:param str program:
The name of the program/application that we're executing.
:param str version:
The version of the program/application we're executing.
"""
#: The timestamp when the Application instance was instantiated.
self.start_time = time.time()
#: The timestamp when the Application finished reported errors.
self.end_time: Optional[float] = None
#: The name of the program being run
self.program = program
#: The version of the program being run
self.version = version
#: The prelimary argument parser for handling options required for
#: obtaining and parsing the configuration file.
self.prelim_arg_parser = argparse.ArgumentParser(add_help=False)
options.register_preliminary_options(self.prelim_arg_parser)
#: The instance of :class:`flake8.options.manager.OptionManager` used
#: to parse and handle the options and arguments passed by the user
self.option_manager = manager.OptionManager(
prog="flake8",
version=flake8.__version__,
parents=[self.prelim_arg_parser],
)
options.register_default_options(self.option_manager)
#: The instance of :class:`flake8.plugins.manager.Checkers`
self.check_plugins: Optional[plugin_manager.Checkers] = None
#: The instance of :class:`flake8.plugins.manager.ReportFormatters`
self.formatting_plugins: Optional[
plugin_manager.ReportFormatters
] = None
#: The user-selected formatter from :attr:`formatting_plugins`
self.formatter: Optional[BaseFormatter] = None
#: The :class:`flake8.style_guide.StyleGuideManager` built from the
#: user's options
self.guide: Optional[style_guide.StyleGuideManager] = None
#: The :class:`flake8.checker.Manager` that will handle running all of
#: the checks selected by the user.
self.file_checker_manager: Optional[checker.Manager] = None
#: The user-supplied options parsed into an instance of
#: :class:`argparse.Namespace`
self.options: Optional[argparse.Namespace] = None
#: The left over arguments that were not parsed by
#: :attr:`option_manager`
self.args: Optional[List[str]] = None
#: The number of errors, warnings, and other messages after running
#: flake8 and taking into account ignored errors and lines.
self.result_count = 0
#: The total number of errors before accounting for ignored errors and
#: lines.
self.total_result_count = 0
#: Whether or not something catastrophic happened and we should exit
#: with a non-zero status code
self.catastrophic_failure = False
#: Whether the program is processing a diff or not
self.running_against_diff = False
#: The parsed diff information
self.parsed_diff: Dict[str, Set[int]] = {}
def parse_preliminary_options(
self, argv: List[str]
) -> Tuple[argparse.Namespace, List[str]]:
"""Get preliminary options from the CLI, pre-plugin-loading.
We need to know the values of a few standard options so that we can
locate configuration files and configure logging.
Since plugins aren't loaded yet, there may be some as-yet-unknown
options; we ignore those for now, they'll be parsed later when we do
real option parsing.
:param list argv:
Command-line arguments passed in directly.
:returns:
Populated namespace and list of remaining argument strings.
:rtype:
(argparse.Namespace, list)
"""
args, rest = self.prelim_arg_parser.parse_known_args(argv)
# XXX (ericvw): Special case "forwarding" the output file option so
# that it can be reparsed again for the BaseFormatter.filename.
if args.output_file:
rest.extend(("--output-file", args.output_file))
return args, rest
def exit(self) -> None:
"""Handle finalization and exiting the program.
This should be the last thing called on the application instance. It
will check certain options and exit appropriately.
"""
assert self.options is not None
if self.options.count:
print(self.result_count)
if self.options.exit_zero:
raise SystemExit(self.catastrophic_failure)
else:
raise SystemExit(
(self.result_count > 0) or self.catastrophic_failure
)
def find_plugins(self, config_finder: config.ConfigFileFinder) -> None:
"""Find and load the plugins for this application.
Set the :attr:`check_plugins` and :attr:`formatting_plugins` attributes
based on the discovered plugins found.
:param config.ConfigFileFinder config_finder:
The finder for finding and reading configuration files.
"""
local_plugins = config.get_local_plugins(config_finder)
sys.path.extend(local_plugins.paths)
self.check_plugins = plugin_manager.Checkers(local_plugins.extension)
self.formatting_plugins = plugin_manager.ReportFormatters(
local_plugins.report
)
self.check_plugins.load_plugins()
self.formatting_plugins.load_plugins()
def register_plugin_options(self) -> None:
"""Register options provided by plugins to our option manager."""
assert self.check_plugins is not None
self.check_plugins.register_options(self.option_manager)
self.check_plugins.register_plugin_versions(self.option_manager)
assert self.formatting_plugins is not None
self.formatting_plugins.register_options(self.option_manager)
def parse_configuration_and_cli(
self,
config_finder: config.ConfigFileFinder,
argv: List[str],
) -> None:
"""Parse configuration files and the CLI options.
:param config.ConfigFileFinder config_finder:
The finder for finding and reading configuration files.
:param list argv:
Command-line arguments passed in directly.
"""
self.options, self.args = aggregator.aggregate_options(
self.option_manager,
config_finder,
argv,
)
self.running_against_diff = self.options.diff
if self.running_against_diff:
self.parsed_diff = utils.parse_unified_diff()
if not self.parsed_diff:
self.exit()
assert self.check_plugins is not None
self.check_plugins.provide_options(
self.option_manager, self.options, self.args
)
assert self.formatting_plugins is not None
self.formatting_plugins.provide_options(
self.option_manager, self.options, self.args
)
def formatter_for(self, formatter_plugin_name):
"""Retrieve the formatter class by plugin name."""
assert self.formatting_plugins is not None
default_formatter = self.formatting_plugins["default"]
formatter_plugin = self.formatting_plugins.get(formatter_plugin_name)
if formatter_plugin is None:
LOG.warning(
'"%s" is an unknown formatter. Falling back to default.',
formatter_plugin_name,
)
formatter_plugin = default_formatter
return formatter_plugin.execute
def make_formatter(
self, formatter_class: Optional[Type["BaseFormatter"]] = None
) -> None:
"""Initialize a formatter based on the parsed options."""
assert self.options is not None
format_plugin = self.options.format
if 1 <= self.options.quiet < 2:
format_plugin = "quiet-filename"
elif 2 <= self.options.quiet:
format_plugin = "quiet-nothing"
if formatter_class is None:
formatter_class = self.formatter_for(format_plugin)
self.formatter = formatter_class(self.options)
def make_guide(self) -> None:
"""Initialize our StyleGuide."""
assert self.formatter is not None
assert self.options is not None
self.guide = style_guide.StyleGuideManager(
self.options, self.formatter
)
if self.running_against_diff:
self.guide.add_diff_ranges(self.parsed_diff)
def make_file_checker_manager(self) -> None:
"""Initialize our FileChecker Manager."""
self.file_checker_manager = checker.Manager(
style_guide=self.guide,
arguments=self.args,
checker_plugins=self.check_plugins,
)
def run_checks(self, files: Optional[List[str]] = None) -> None:
"""Run the actual checks with the FileChecker Manager.
This method encapsulates the logic to make a
:class:`~flake8.checker.Manger` instance run the checks it is
managing.
:param list files:
List of filenames to process
"""
assert self.file_checker_manager is not None
if self.running_against_diff:
files = sorted(self.parsed_diff)
self.file_checker_manager.start(files)
try:
self.file_checker_manager.run()
except exceptions.PluginExecutionFailed as plugin_failed:
print(str(plugin_failed))
print("Run flake8 with greater verbosity to see more details")
self.catastrophic_failure = True
LOG.info("Finished running")
self.file_checker_manager.stop()
self.end_time = time.time()
def report_benchmarks(self):
"""Aggregate, calculate, and report benchmarks for this run."""
assert self.options is not None
if not self.options.benchmark:
return
assert self.file_checker_manager is not None
assert self.end_time is not None
time_elapsed = self.end_time - self.start_time
statistics = [("seconds elapsed", time_elapsed)]
add_statistic = statistics.append
for statistic in defaults.STATISTIC_NAMES + ("files",):
value = self.file_checker_manager.statistics[statistic]
total_description = f"total {statistic} processed"
add_statistic((total_description, value))
per_second_description = f"{statistic} processed per second"
add_statistic((per_second_description, int(value / time_elapsed)))
assert self.formatter is not None
self.formatter.show_benchmarks(statistics)
def report_errors(self) -> None:
"""Report all the errors found by flake8 3.0.
This also updates the :attr:`result_count` attribute with the total
number of errors, warnings, and other messages found.
"""
LOG.info("Reporting errors")
assert self.file_checker_manager is not None
results = self.file_checker_manager.report()
self.total_result_count, self.result_count = results
LOG.info(
"Found a total of %d violations and reported %d",
self.total_result_count,
self.result_count,
)
def report_statistics(self):
"""Aggregate and report statistics from this run."""
assert self.options is not None
if not self.options.statistics:
return
assert self.formatter is not None
assert self.guide is not None
self.formatter.show_statistics(self.guide.stats)
def initialize(self, argv: List[str]) -> None:
"""Initialize the application to be run.
This finds the plugins, registers their options, and parses the
command-line arguments.
"""
# NOTE(sigmavirus24): When updating this, make sure you also update
# our legacy API calls to these same methods.
prelim_opts, remaining_args = self.parse_preliminary_options(argv)
flake8.configure_logging(prelim_opts.verbose, prelim_opts.output_file)
config_finder = config.ConfigFileFinder(
self.program,
prelim_opts.append_config,
config_file=prelim_opts.config,
ignore_config_files=prelim_opts.isolated,
)
self.find_plugins(config_finder)
self.register_plugin_options()
self.parse_configuration_and_cli(
config_finder,
remaining_args,
)
self.make_formatter()
self.make_guide()
self.make_file_checker_manager()
def report(self):
"""Report errors, statistics, and benchmarks."""
assert self.formatter is not None
self.formatter.start()
self.report_errors()
self.report_statistics()
self.report_benchmarks()
self.formatter.stop()
def _run(self, argv: List[str]) -> None:
self.initialize(argv)
self.run_checks()
self.report()
def run(self, argv: List[str]) -> None:
"""Run our application.
This method will also handle KeyboardInterrupt exceptions for the
entirety of the flake8 application. If it sees a KeyboardInterrupt it
will forcibly clean up the :class:`~flake8.checker.Manager`.
"""
try:
self._run(argv)
except KeyboardInterrupt as exc:
print("... stopped")
LOG.critical("Caught keyboard interrupt from user")
LOG.exception(exc)
self.catastrophic_failure = True
except exceptions.ExecutionError as exc:
print("There was a critical error during execution of Flake8:")
print(exc)
LOG.exception(exc)
self.catastrophic_failure = True
except exceptions.EarlyQuit:
self.catastrophic_failure = True
print("... stopped while processing files")

View file

@ -0,0 +1,23 @@
"""Command-line implementation of flake8."""
import sys
from typing import List
from typing import Optional
from flake8.main import application
def main(argv: Optional[List[str]] = None) -> None:
"""Execute the main bit of the application.
This handles the creation of an instance of :class:`Application`, runs it,
and then exits the application.
:param list argv:
The arguments to be passed to the application for parsing.
"""
if argv is None:
argv = sys.argv[1:]
app = application.Application()
app.run(argv)
app.exit()

View file

@ -0,0 +1,64 @@
"""Module containing the logic for our debugging logic."""
import argparse
import json
import platform
from typing import Dict
from typing import List
class DebugAction(argparse.Action):
"""argparse action to print debug information."""
def __init__(self, *args, **kwargs):
"""Initialize the action.
This takes an extra `option_manager` keyword argument which will be
used to delay response.
"""
self._option_manager = kwargs.pop("option_manager")
super().__init__(*args, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
"""Perform the argparse action for printing debug information."""
# NOTE(sigmavirus24): Flake8 parses options twice. The first time, we
# will not have any registered plugins. We can skip this one and only
# take action on the second time we're called.
if not self._option_manager.registered_plugins:
return
print(
json.dumps(
information(self._option_manager), indent=2, sort_keys=True
)
)
raise SystemExit(0)
def information(option_manager):
"""Generate the information to be printed for the bug report."""
return {
"version": option_manager.version,
"plugins": plugins_from(option_manager),
"dependencies": dependencies(),
"platform": {
"python_implementation": platform.python_implementation(),
"python_version": platform.python_version(),
"system": platform.system(),
},
}
def plugins_from(option_manager):
"""Generate the list of plugins installed."""
return [
{
"plugin": plugin.name,
"version": plugin.version,
"is_local": plugin.local,
}
for plugin in sorted(option_manager.registered_plugins)
]
def dependencies() -> List[Dict[str, str]]:
"""Generate the list of dependencies we care about."""
return []

View file

@ -0,0 +1,374 @@
"""Contains the logic for all of the default options for Flake8."""
import argparse
import functools
from flake8 import defaults
from flake8.main import debug
def register_preliminary_options(parser: argparse.ArgumentParser) -> None:
"""Register the preliminary options on our OptionManager.
The preliminary options include:
- ``-v``/``--verbose``
- ``--output-file``
- ``--append-config``
- ``--config``
- ``--isolated``
"""
add_argument = parser.add_argument
add_argument(
"-v",
"--verbose",
default=0,
action="count",
help="Print more information about what is happening in flake8."
" This option is repeatable and will increase verbosity each "
"time it is repeated.",
)
add_argument(
"--output-file", default=None, help="Redirect report to a file."
)
# Config file options
add_argument(
"--append-config",
action="append",
help="Provide extra config files to parse in addition to the files "
"found by Flake8 by default. These files are the last ones read "
"and so they take the highest precedence when multiple files "
"provide the same option.",
)
add_argument(
"--config",
default=None,
help="Path to the config file that will be the authoritative config "
"source. This will cause Flake8 to ignore all other "
"configuration files.",
)
add_argument(
"--isolated",
default=False,
action="store_true",
help="Ignore all configuration files.",
)
class JobsArgument:
"""Type callback for the --jobs argument."""
def __init__(self, arg: str) -> None:
"""Parse and validate the --jobs argument.
:param str arg:
The argument passed by argparse for validation
"""
self.is_auto = False
self.n_jobs = -1
if arg == "auto":
self.is_auto = True
elif arg.isdigit():
self.n_jobs = int(arg)
else:
raise argparse.ArgumentTypeError(
f"{arg!r} must be 'auto' or an integer.",
)
def __str__(self):
"""Format our JobsArgument class."""
return "auto" if self.is_auto else str(self.n_jobs)
def register_default_options(option_manager):
"""Register the default options on our OptionManager.
The default options include:
- ``-q``/``--quiet``
- ``--count``
- ``--diff``
- ``--exclude``
- ``--extend-exclude``
- ``--filename``
- ``--format``
- ``--hang-closing``
- ``--ignore``
- ``--extend-ignore``
- ``--per-file-ignores``
- ``--max-line-length``
- ``--max-doc-length``
- ``--indent-size``
- ``--select``
- ``--extend-select``
- ``--disable-noqa``
- ``--show-source``
- ``--statistics``
- ``--enable-extensions``
- ``--exit-zero``
- ``-j``/``--jobs``
- ``--tee``
- ``--benchmark``
- ``--bug-report``
"""
add_option = option_manager.add_option
# pep8 options
add_option(
"-q",
"--quiet",
default=0,
action="count",
parse_from_config=True,
help="Report only file names, or nothing. This option is repeatable.",
)
add_option(
"--count",
action="store_true",
parse_from_config=True,
help="Print total number of errors and warnings to standard error and"
" set the exit code to 1 if total is not empty.",
)
add_option(
"--diff",
action="store_true",
help="Report changes only within line number ranges in the unified "
"diff provided on standard in by the user.",
)
add_option(
"--exclude",
metavar="patterns",
default=",".join(defaults.EXCLUDE),
comma_separated_list=True,
parse_from_config=True,
normalize_paths=True,
help="Comma-separated list of files or directories to exclude."
" (Default: %(default)s)",
)
add_option(
"--extend-exclude",
metavar="patterns",
default="",
parse_from_config=True,
comma_separated_list=True,
normalize_paths=True,
help="Comma-separated list of files or directories to add to the list"
" of excluded ones.",
)
add_option(
"--filename",
metavar="patterns",
default="*.py",
parse_from_config=True,
comma_separated_list=True,
help="Only check for filenames matching the patterns in this comma-"
"separated list. (Default: %(default)s)",
)
add_option(
"--stdin-display-name",
default="stdin",
help="The name used when reporting errors from code passed via stdin."
" This is useful for editors piping the file contents to flake8."
" (Default: %(default)s)",
)
# TODO(sigmavirus24): Figure out --first/--repeat
# NOTE(sigmavirus24): We can't use choices for this option since users can
# freely provide a format string and that will break if we restrict their
# choices.
add_option(
"--format",
metavar="format",
default="default",
parse_from_config=True,
help="Format errors according to the chosen formatter.",
)
add_option(
"--hang-closing",
action="store_true",
parse_from_config=True,
help="Hang closing bracket instead of matching indentation of opening"
" bracket's line.",
)
add_option(
"--ignore",
metavar="errors",
default=",".join(defaults.IGNORE),
parse_from_config=True,
comma_separated_list=True,
help="Comma-separated list of errors and warnings to ignore (or skip)."
" For example, ``--ignore=E4,E51,W234``. (Default: %(default)s)",
)
add_option(
"--extend-ignore",
metavar="errors",
default="",
parse_from_config=True,
comma_separated_list=True,
help="Comma-separated list of errors and warnings to add to the list"
" of ignored ones. For example, ``--extend-ignore=E4,E51,W234``.",
)
add_option(
"--per-file-ignores",
default="",
parse_from_config=True,
help="A pairing of filenames and violation codes that defines which "
"violations to ignore in a particular file. The filenames can be "
"specified in a manner similar to the ``--exclude`` option and the "
"violations work similarly to the ``--ignore`` and ``--select`` "
"options.",
)
add_option(
"--max-line-length",
type=int,
metavar="n",
default=defaults.MAX_LINE_LENGTH,
parse_from_config=True,
help="Maximum allowed line length for the entirety of this run. "
"(Default: %(default)s)",
)
add_option(
"--max-doc-length",
type=int,
metavar="n",
default=None,
parse_from_config=True,
help="Maximum allowed doc line length for the entirety of this run. "
"(Default: %(default)s)",
)
add_option(
"--indent-size",
type=int,
metavar="n",
default=defaults.INDENT_SIZE,
parse_from_config=True,
help="Number of spaces used for indentation (Default: %(default)s)",
)
add_option(
"--select",
metavar="errors",
default=",".join(defaults.SELECT),
parse_from_config=True,
comma_separated_list=True,
help="Comma-separated list of errors and warnings to enable."
" For example, ``--select=E4,E51,W234``. (Default: %(default)s)",
)
add_option(
"--extend-select",
metavar="errors",
default="",
parse_from_config=True,
comma_separated_list=True,
help=(
"Comma-separated list of errors and warnings to add to the list "
"of selected ones. For example, ``--extend-select=E4,E51,W234``."
),
)
add_option(
"--disable-noqa",
default=False,
parse_from_config=True,
action="store_true",
help='Disable the effect of "# noqa". This will report errors on '
'lines with "# noqa" at the end.',
)
# TODO(sigmavirus24): Decide what to do about --show-pep8
add_option(
"--show-source",
action="store_true",
parse_from_config=True,
help="Show the source generate each error or warning.",
)
add_option(
"--no-show-source",
action="store_false",
dest="show_source",
parse_from_config=False,
help="Negate --show-source",
)
add_option(
"--statistics",
action="store_true",
parse_from_config=True,
help="Count errors and warnings.",
)
# Flake8 options
add_option(
"--enable-extensions",
default="",
parse_from_config=True,
comma_separated_list=True,
help="Enable plugins and extensions that are otherwise disabled "
"by default",
)
add_option(
"--exit-zero",
action="store_true",
help='Exit with status code "0" even if there are errors.',
)
add_option(
"-j",
"--jobs",
default="auto",
parse_from_config=True,
type=JobsArgument,
help="Number of subprocesses to use to run checks in parallel. "
'This is ignored on Windows. The default, "auto", will '
"auto-detect the number of processors available to use."
" (Default: %(default)s)",
)
add_option(
"--tee",
default=False,
parse_from_config=True,
action="store_true",
help="Write to stdout and output-file.",
)
# Benchmarking
add_option(
"--benchmark",
default=False,
action="store_true",
help="Print benchmark information about this run of Flake8",
)
# Debugging
add_option(
"--bug-report",
action=functools.partial(
debug.DebugAction, option_manager=option_manager
),
nargs=0,
help="Print information necessary when preparing a bug report",
)