PY-19353 PY-19020 Update pep8.py to version 2.0.0

This version contains long-awaited fixes for the issues #496 and #507. The script itself is called pycodestyle.py now.
2026-09-27 10:03:11 +07:00 · 2016-06-02 17:46:16 +03:00
parent 397fe99a92
commit 2b337083c7
3 changed files with 122 additions and 55 deletions
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
-# pep8.py - Check Python source code formatting, according to PEP 8
+# pycodestyle.py - Check Python source code formatting, according to PEP 8
+#
 # Copyright (C) 2006-2009 Johann C. Rocholl <johann@rocholl.net>
 # Copyright (C) 2009-2014 Florent Xicluna <florent.xicluna@gmail.com>
-# Copyright (C) 2014 Ian Lee <ianlee1521@gmail.com>
+# Copyright (C) 2014-2016 Ian Lee <ianlee1521@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation files
@@ -28,10 +29,10 @@ r"""
 Check Python source code formatting, according to PEP 8.

 For usage and a list of options, try this:
-$ python pep8.py -h
+$ python pycodestyle.py -h

 This program and its regression test suite live here:
-http://github.com/jcrocholl/pep8
+https://github.com/pycqa/pycodestyle

 Groups of errors and warnings:
 E errors
@@ -54,6 +55,7 @@ import time
 import inspect
 import keyword
 import tokenize
+import warnings
 from optparse import OptionParser
 from fnmatch import fnmatch
 try:
@@ -62,10 +64,10 @@ try:
 except ImportError:
    from ConfigParser import RawConfigParser

-__version__ = '1.6.2'
+__version__ = '2.0.0'

 DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox'
-DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704'
+DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704,W503'
 try:
    if sys.platform == 'win32':
        USER_CONFIG = os.path.expanduser(r'~\.pep8')
@@ -77,7 +79,7 @@ try:
 except ImportError:
    USER_CONFIG = None

-PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8')
+PROJECT_CONFIG = ('setup.cfg', 'tox.ini')
 TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite')
 MAX_LINE_LENGTH = 79
 REPORT_FORMAT = {
@@ -108,7 +110,7 @@ ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b')
 DOCSTRING_REGEX = re.compile(r'u?r?["\']')
 EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]')
 WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?:  |\t)')
-COMPARE_SINGLETON_REGEX = re.compile(r'\b(None|False|True)?\s*([=!]=)'
+COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)'
                                     r'\s*(?(1)|(None|False|True))\b')
 COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s')
 COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type'
@@ -325,6 +327,23 @@ def whitespace_around_keywords(logical_line):
            yield match.start(2), "E271 multiple spaces after keyword"


+def missing_whitespace_after_import_keyword(logical_line):
+    r"""Multiple imports in form from x import (a, b, c) should have space
+    between import statement and parenthesised name list.
+
+    Okay: from foo import (bar, baz)
+    E275: from foo import(bar, baz)
+    E275: from importable.module import(bar, baz)
+    """
+    line = logical_line
+    indicator = ' import('
+    if line.startswith('from '):
+        found = line.find(indicator)
+        if -1 < found:
+            pos = found + len(indicator) - 1
+            yield pos, "E275 missing whitespace after keyword"
+
+
 def missing_whitespace(logical_line):
    r"""Each comma, semicolon or colon should be followed by whitespace.

@@ -565,7 +584,7 @@ def continued_indentation(logical_line, tokens, indent_level, hang_closing,
                        break
            assert len(indent) == depth + 1
            if start[1] not in indent_chances:
-                # allow to line up tokens
+                # allow lining up tokens
                indent_chances[start[1]] = text

        last_token_multiline = (start[0] != end[0])
@@ -759,6 +778,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens):
    Okay: boolean(a <= b)
    Okay: boolean(a >= b)
    Okay: def foo(arg: int = 42):
+    Okay: async def foo(arg: int = 42):

    E251: def complex(real, imag = 0.0):
    E251: return magic(r = real, i = imag)
@@ -767,7 +787,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens):
    no_space = False
    prev_end = None
    annotated_func_arg = False
-    in_def = logical_line.startswith('def')
+    in_def = logical_line.startswith(('def', 'async def'))
    message = "E251 unexpected spaces around keyword / parameter equals"
    for token_type, text, start, end, line in tokens:
        if token_type == tokenize.NL:
@@ -777,9 +797,9 @@ def whitespace_around_named_parameter_equals(logical_line, tokens):
            if start != prev_end:
                yield (prev_end, message)
        if token_type == tokenize.OP:
-            if text == '(':
+            if text in '([':
                parens += 1
-            elif text == ')':
+            elif text in ')]':
                parens -= 1
            elif in_def and text == ':' and parens == 1:
                annotated_func_arg = True
@@ -837,7 +857,7 @@ def whitespace_before_comment(logical_line, tokens):


 def imports_on_separate_lines(logical_line):
-    r"""Imports should usually be on separate lines.
+    r"""Place imports on separate lines.

    Okay: import os\nimport sys
    E401: import sys, os
@@ -857,8 +877,10 @@ def imports_on_separate_lines(logical_line):

 def module_imports_on_top_of_file(
        logical_line, indent_level, checker_state, noqa):
-    r"""Imports are always put at the top of the file, just after any module
-    comments and docstrings, and before module globals and constants.
+    r"""Place imports at the top of the file.
+
+    Always put imports at the top of the file, just after any module comments
+    and docstrings, and before module globals and constants.

    Okay: import os
    Okay: # this is a comment\nimport os
@@ -1017,16 +1039,22 @@ def break_around_binary_operator(logical_line, tokens):
    Okay: x = '''\n''' + ''
    Okay: foo(x,\n    -y)
    Okay: foo(x,  # comment\n    -y)
+    Okay: var = (1 &\n       ~2)
+    Okay: var = (1 /\n       -2)
+    Okay: var = (1 +\n       -1 +\n       -2)
    """
    def is_binary_operator(token_type, text):
        # The % character is strictly speaking a binary operator, but the
        # common usage seems to be to put it next to the format parameters,
        # after a line break.
        return ((token_type == tokenize.OP or text in ['and', 'or']) and
-                text not in "()[]{},:.;@=%")
+                text not in "()[]{},:.;@=%~")

    line_break = False
    unary_context = True
+    # Previous non-newline token types and text
+    previous_token_type = None
+    previous_text = None
    for token_type, text, start, end, line in tokens:
        if token_type == tokenize.COMMENT:
            continue
@@ -1034,10 +1062,14 @@ def break_around_binary_operator(logical_line, tokens):
            line_break = True
        else:
            if (is_binary_operator(token_type, text) and line_break and
-                    not unary_context):
+                    not unary_context and
+                    not is_binary_operator(previous_token_type,
+                                           previous_text)):
                yield start, "W503 line break before binary operator"
            unary_context = text in '([{,;'
            line_break = False
+            previous_token_type = token_type
+            previous_text = text


 def comparison_to_singleton(logical_line, noqa):
@@ -1156,7 +1188,7 @@ def python_3000_not_equal(logical_line):


 def python_3000_backticks(logical_line):
-    r"""Backticks are removed in Python 3: use repr() instead.
+    r"""Use repr() instead of backticks in Python 3.

    Okay: val = repr(1 + 2)
    W604: val = `1 + 2`
@@ -1171,7 +1203,7 @@ def python_3000_backticks(logical_line):
 ##############################################################################


-if '' == ''.encode():
+if sys.version_info < (3,):
    # Python 2: implicit encoding.
    def readlines(filename):
        """Read the source code."""
@@ -1195,7 +1227,9 @@ else:
    isidentifier = str.isidentifier

    def stdin_get_value():
+        """Read the value from stdin."""
        return TextIOWrapper(sys.stdin.buffer, errors='ignore').read()
+
 noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search


@@ -1314,6 +1348,16 @@ if COMMENT_WITH_NL:
 _checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}}


+def _get_parameters(function):
+    if sys.version_info >= (3, 3):
+        return [parameter.name
+                for parameter
+                in inspect.signature(function).parameters.values()
+                if parameter.kind == parameter.POSITIONAL_OR_KEYWORD]
+    else:
+        return inspect.getargspec(function)[0]
+
+
 def register_check(check, codes=None):
    """Register a new check object."""
    def _add_check(check, kind, codes, args):
@@ -1322,13 +1366,13 @@ def register_check(check, codes=None):
        else:
            _checks[kind][check] = (codes or [''], args)
    if inspect.isfunction(check):
-        args = inspect.getargspec(check)[0]
+        args = _get_parameters(check)
        if args and args[0] in ('physical_line', 'logical_line'):
            if codes is None:
                codes = ERRORCODE_REGEX.findall(check.__doc__ or '')
            _add_check(check, args[0], codes, args)
    elif inspect.isclass(check):
-        if inspect.getargspec(check.__init__)[0][:2] == ['self', 'tree']:
+        if _get_parameters(check.__init__)[:2] == ['self', 'tree']:
            _add_check(check, 'tree', codes, None)


@@ -1419,7 +1463,7 @@ class Checker(object):
        return check(*arguments)

    def init_checker_state(self, name, argument_names):
-        """ Prepares a custom state for the specific checker plugin."""
+        """Prepare custom state for the specific checker plugin."""
        if 'checker_state' in argument_names:
            self.checker_state = self._checker_states.setdefault(name, {})

@@ -1504,7 +1548,7 @@ class Checker(object):
        """Build the file's AST and run all AST checks."""
        try:
            tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST)
-        except (SyntaxError, TypeError):
+        except (ValueError, SyntaxError, TypeError):
            return self.report_invalid_syntax()
        for name, cls, __ in self._ast_checks:
            checker = cls(tree, self.filename)
@@ -1701,6 +1745,7 @@ class BaseReport(object):

 class FileReport(BaseReport):
    """Collect the results of the checks and print only the filenames."""
+
    print_filename = True


@@ -1908,6 +1953,7 @@ class StyleGuide(object):


 def get_parser(prog='pep8', version=__version__):
+    """Create the parser for the program."""
    parser = OptionParser(prog=prog, version=version,
                          usage="%prog [options] input ...")
    parser.config_options = [
@@ -1955,8 +2001,8 @@ def get_parser(prog='pep8', version=__version__):
    parser.add_option('--format', metavar='format', default='default',
                      help="set the error format [default|pylint|<custom>]")
    parser.add_option('--diff', action='store_true',
-                      help="report only lines changed according to the "
-                           "unified diff received on STDIN")
+                      help="report changes only within line number ranges in "
+                           "the unified diff received on STDIN")
    group = parser.add_option_group("Testing Options")
    if os.path.exists(TESTSUITE_PATH):
        group.add_option('--testsuite', metavar='dir',
@@ -1969,7 +2015,7 @@ def get_parser(prog='pep8', version=__version__):


 def read_config(options, args, arglist, parser):
-    """Read and parse configurations
+    """Read and parse configurations.

    If a config file is specified on the command line with the "--config"
    option, then only it is used for configuration.
@@ -1984,24 +2030,24 @@ def read_config(options, args, arglist, parser):

    local_dir = os.curdir

+    if USER_CONFIG and os.path.isfile(USER_CONFIG):
+        if options.verbose:
+            print('user configuration: %s' % USER_CONFIG)
+        config.read(USER_CONFIG)
+
+    parent = tail = args and os.path.abspath(os.path.commonprefix(args))
+    while tail:
+        if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG):
+            local_dir = parent
+            if options.verbose:
+                print('local configuration: in %s' % parent)
+            break
+        (parent, tail) = os.path.split(parent)
+
    if cli_conf and os.path.isfile(cli_conf):
        if options.verbose:
            print('cli configuration: %s' % cli_conf)
        config.read(cli_conf)
-    else:
-        if USER_CONFIG and os.path.isfile(USER_CONFIG):
-            if options.verbose:
-                print('user configuration: %s' % USER_CONFIG)
-            config.read(USER_CONFIG)
-
-        parent = tail = args and os.path.abspath(os.path.commonprefix(args))
-        while tail:
-            if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG):
-                local_dir = parent
-                if options.verbose:
-                    print('local configuration: in %s' % parent)
-                break
-            (parent, tail) = os.path.split(parent)

    pep8_section = parser.prog
    if config.has_section(pep8_section):
@@ -2074,10 +2120,10 @@ def process_options(arglist=None, parse_argv=False, config_file=None,
        options = read_config(options, args, arglist, parser)
        options.reporter = parse_argv and options.quiet == 1 and FileReport

-    options.filename = options.filename and options.filename.split(',')
+    options.filename = _parse_multi_options(options.filename)
    options.exclude = normalize_paths(options.exclude)
-    options.select = options.select and options.select.split(',')
-    options.ignore = options.ignore and options.ignore.split(',')
+    options.select = _parse_multi_options(options.select)
+    options.ignore = _parse_multi_options(options.ignore)

    if options.diff:
        options.reporter = DiffReport
@@ -2088,6 +2134,22 @@ def process_options(arglist=None, parse_argv=False, config_file=None,
    return options, args


+def _parse_multi_options(options, split_token=','):
+    r"""Split and strip and discard empties.
+
+    Turns the following:
+
+    A,
+    B,
+
+    into ["A", "B"]
+    """
+    if options:
+        return [o.strip() for o in options.split(split_token) if o.strip()]
+    else:
+        return options
+
+
 def _main():
    """Parse options and run checks on Python source."""
    import signal
@@ -2098,23 +2160,29 @@ def _main():
    except AttributeError:
        pass    # not supported on Windows

-    pep8style = StyleGuide(parse_argv=True)
-    options = pep8style.options
+    style_guide = StyleGuide(parse_argv=True)
+    options = style_guide.options
+
    if options.doctest or options.testsuite:
        from testsuite.support import run_tests
-        report = run_tests(pep8style)
+        report = run_tests(style_guide)
    else:
-        report = pep8style.check_files()
+        report = style_guide.check_files()
+
    if options.statistics:
        report.print_statistics()
+
    if options.benchmark:
        report.print_benchmark()
+
    if options.testsuite and not options.quiet:
        report.print_results()
+
    if report.total_errors:
        if options.count:
            sys.stderr.write(str(report.total_errors) + '\n')
        sys.exit(1)

+
 if __name__ == '__main__':
    _main()
@@ -76,7 +76,7 @@ public enum PythonHelper implements HelperPackage {
  EXTRA_SYSPATH("extra_syspath.py"),
  SYSPATH("syspath.py"),

-  PEP8("pep8.py"),
+  PYCODESTYLE("pycodestyle.py"),

  REST_RUNNER("rest_runners/rst2smth.py"),

@@ -37,7 +37,6 @@ import com.intellij.openapi.editor.ex.EditorSettingsExternalizable;
 import com.intellij.openapi.module.ModuleUtilCore;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.projectRoots.Sdk;
-import com.intellij.openapi.util.Computable;
 import com.intellij.openapi.util.TextRange;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
@@ -169,7 +168,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator<Pep8ExternalAnnotat
  }

  private static void reportMissingInterpreter() {
-    LOG.info("Found no suitable interpreter to run pep8.py. Available interpreters are: [");
+    LOG.info("Found no suitable interpreter to run pycodestyle.py. Available interpreters are: [");
    List<Sdk> allSdks = PythonSdkType.getAllSdks();
    Collections.sort(allSdks, PreferredSdkComparator.INSTANCE);
    for (Sdk sdk : allSdks) {
@@ -190,7 +189,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator<Pep8ExternalAnnotat
    options.add("--max-line-length=" + collectedInfo.margin);
    options.add("-");

-    GeneralCommandLine cmd = PythonHelper.PEP8.newCommandLine(collectedInfo.interpreterPath, options);
+    GeneralCommandLine cmd = PythonHelper.PYCODESTYLE.newCommandLine(collectedInfo.interpreterPath, options);

    ProcessOutput output = PySdkUtil.getProcessOutput(cmd, new File(collectedInfo.interpreterPath).getParent(),
                                                      ImmutableMap.of("PYTHONBUFFERED", "1"),
@@ -199,7 +198,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator<Pep8ExternalAnnotat

    Results results = new Results(collectedInfo.level);
    if (output.isTimeout()) {
-      LOG.info("Timeout running pep8.py");
+      LOG.info("Timeout running pycodestyle.py");
    }
    else if (output.getStderrLines().isEmpty()) {
      for (String line : output.getStdoutLines()) {
@@ -210,7 +209,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator<Pep8ExternalAnnotat
      }
    }
    else if (((ApplicationInfoImpl) ApplicationInfo.getInstance()).isEAP()) {
-      LOG.info("Error running pep8.py: " + output.getStderr());
+      LOG.info("Error running pycodestyle.py: " + output.getStderr());
    }
    return results;
  }
@@ -253,7 +252,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator<Pep8ExternalAnnotat
      if (problemElement != null) {
        TextRange problemRange = problemElement.getTextRange();
        // Multi-line warnings are shown only in the gutter and it's not the desired behavior from the usability point of view.
-        // So we register it only on that line where pep8.py found the problem originally.
+        // So we register it only on that line where pycodestyle.py found the problem originally.
        if (crossesLineBoundary(document, text, problemRange)) {
          final int lineEndOffset;
          if (document != null) {
@@ -371,7 +370,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator<Pep8ExternalAnnotat
      return new Problem(line, column, m.group(3), m.group(4));
    }
    if (((ApplicationInfoImpl) ApplicationInfo.getInstance()).isEAP()) {
-      LOG.info("Failed to parse problem line from pep8.py: " + s);
+      LOG.info("Failed to parse problem line from pycodestyle.py: " + s);
    }
    return null;
  }