From 2b337083c781746ca227c1b4fada88e4b2c32ca0 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Thu, 2 Jun 2016 16:00:41 +0300 Subject: [PATCH] PY-19353 PY-19020 Update pep8.py to version 2.0.0 This version contains long-awaited fixes for the issues #496 and #507. The script itself is called pycodestyle.py now. --- python/helpers/{pep8.py => pycodestyle.py} | 162 +++++++++++++----- .../com/jetbrains/python/PythonHelper.java | 2 +- .../validation/Pep8ExternalAnnotator.java | 13 +- 3 files changed, 122 insertions(+), 55 deletions(-) rename python/helpers/{pep8.py => pycodestyle.py} (95%) diff --git a/python/helpers/pep8.py b/python/helpers/pycodestyle.py similarity index 95% rename from python/helpers/pep8.py rename to python/helpers/pycodestyle.py index d7907e532f0b..70263968d7c5 100755 --- a/python/helpers/pep8.py +++ b/python/helpers/pycodestyle.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -# pep8.py - Check Python source code formatting, according to PEP 8 +# pycodestyle.py - Check Python source code formatting, according to PEP 8 +# # Copyright (C) 2006-2009 Johann C. Rocholl # Copyright (C) 2009-2014 Florent Xicluna -# Copyright (C) 2014 Ian Lee +# Copyright (C) 2014-2016 Ian Lee # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation files @@ -28,10 +29,10 @@ r""" Check Python source code formatting, according to PEP 8. For usage and a list of options, try this: -$ python pep8.py -h +$ python pycodestyle.py -h This program and its regression test suite live here: -http://github.com/jcrocholl/pep8 +https://github.com/pycqa/pycodestyle Groups of errors and warnings: E errors @@ -54,6 +55,7 @@ import time import inspect import keyword import tokenize +import warnings from optparse import OptionParser from fnmatch import fnmatch try: @@ -62,10 +64,10 @@ try: except ImportError: from ConfigParser import RawConfigParser -__version__ = '1.6.2' +__version__ = '2.0.0' DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' -DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' +DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704,W503' try: if sys.platform == 'win32': USER_CONFIG = os.path.expanduser(r'~\.pep8') @@ -77,7 +79,7 @@ try: except ImportError: USER_CONFIG = None -PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') +PROJECT_CONFIG = ('setup.cfg', 'tox.ini') TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') MAX_LINE_LENGTH = 79 REPORT_FORMAT = { @@ -108,7 +110,7 @@ ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') DOCSTRING_REGEX = re.compile(r'u?r?["\']') EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') -COMPARE_SINGLETON_REGEX = re.compile(r'\b(None|False|True)?\s*([=!]=)' +COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)' r'\s*(?(1)|(None|False|True))\b') COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s') COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' @@ -325,6 +327,23 @@ def whitespace_around_keywords(logical_line): yield match.start(2), "E271 multiple spaces after keyword" +def missing_whitespace_after_import_keyword(logical_line): + r"""Multiple imports in form from x import (a, b, c) should have space + between import statement and parenthesised name list. + + Okay: from foo import (bar, baz) + E275: from foo import(bar, baz) + E275: from importable.module import(bar, baz) + """ + line = logical_line + indicator = ' import(' + if line.startswith('from '): + found = line.find(indicator) + if -1 < found: + pos = found + len(indicator) - 1 + yield pos, "E275 missing whitespace after keyword" + + def missing_whitespace(logical_line): r"""Each comma, semicolon or colon should be followed by whitespace. @@ -565,7 +584,7 @@ def continued_indentation(logical_line, tokens, indent_level, hang_closing, break assert len(indent) == depth + 1 if start[1] not in indent_chances: - # allow to line up tokens + # allow lining up tokens indent_chances[start[1]] = text last_token_multiline = (start[0] != end[0]) @@ -759,6 +778,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): Okay: boolean(a <= b) Okay: boolean(a >= b) Okay: def foo(arg: int = 42): + Okay: async def foo(arg: int = 42): E251: def complex(real, imag = 0.0): E251: return magic(r = real, i = imag) @@ -767,7 +787,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): no_space = False prev_end = None annotated_func_arg = False - in_def = logical_line.startswith('def') + in_def = logical_line.startswith(('def', 'async def')) message = "E251 unexpected spaces around keyword / parameter equals" for token_type, text, start, end, line in tokens: if token_type == tokenize.NL: @@ -777,9 +797,9 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): if start != prev_end: yield (prev_end, message) if token_type == tokenize.OP: - if text == '(': + if text in '([': parens += 1 - elif text == ')': + elif text in ')]': parens -= 1 elif in_def and text == ':' and parens == 1: annotated_func_arg = True @@ -837,7 +857,7 @@ def whitespace_before_comment(logical_line, tokens): def imports_on_separate_lines(logical_line): - r"""Imports should usually be on separate lines. + r"""Place imports on separate lines. Okay: import os\nimport sys E401: import sys, os @@ -857,8 +877,10 @@ def imports_on_separate_lines(logical_line): def module_imports_on_top_of_file( logical_line, indent_level, checker_state, noqa): - r"""Imports are always put at the top of the file, just after any module - comments and docstrings, and before module globals and constants. + r"""Place imports at the top of the file. + + Always put imports at the top of the file, just after any module comments + and docstrings, and before module globals and constants. Okay: import os Okay: # this is a comment\nimport os @@ -1017,16 +1039,22 @@ def break_around_binary_operator(logical_line, tokens): Okay: x = '''\n''' + '' Okay: foo(x,\n -y) Okay: foo(x, # comment\n -y) + Okay: var = (1 &\n ~2) + Okay: var = (1 /\n -2) + Okay: var = (1 +\n -1 +\n -2) """ def is_binary_operator(token_type, text): # The % character is strictly speaking a binary operator, but the # common usage seems to be to put it next to the format parameters, # after a line break. return ((token_type == tokenize.OP or text in ['and', 'or']) and - text not in "()[]{},:.;@=%") + text not in "()[]{},:.;@=%~") line_break = False unary_context = True + # Previous non-newline token types and text + previous_token_type = None + previous_text = None for token_type, text, start, end, line in tokens: if token_type == tokenize.COMMENT: continue @@ -1034,10 +1062,14 @@ def break_around_binary_operator(logical_line, tokens): line_break = True else: if (is_binary_operator(token_type, text) and line_break and - not unary_context): + not unary_context and + not is_binary_operator(previous_token_type, + previous_text)): yield start, "W503 line break before binary operator" unary_context = text in '([{,;' line_break = False + previous_token_type = token_type + previous_text = text def comparison_to_singleton(logical_line, noqa): @@ -1156,7 +1188,7 @@ def python_3000_not_equal(logical_line): def python_3000_backticks(logical_line): - r"""Backticks are removed in Python 3: use repr() instead. + r"""Use repr() instead of backticks in Python 3. Okay: val = repr(1 + 2) W604: val = `1 + 2` @@ -1171,7 +1203,7 @@ def python_3000_backticks(logical_line): ############################################################################## -if '' == ''.encode(): +if sys.version_info < (3,): # Python 2: implicit encoding. def readlines(filename): """Read the source code.""" @@ -1195,7 +1227,9 @@ else: isidentifier = str.isidentifier def stdin_get_value(): + """Read the value from stdin.""" return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() + noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search @@ -1314,6 +1348,16 @@ if COMMENT_WITH_NL: _checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} +def _get_parameters(function): + if sys.version_info >= (3, 3): + return [parameter.name + for parameter + in inspect.signature(function).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] + else: + return inspect.getargspec(function)[0] + + def register_check(check, codes=None): """Register a new check object.""" def _add_check(check, kind, codes, args): @@ -1322,13 +1366,13 @@ def register_check(check, codes=None): else: _checks[kind][check] = (codes or [''], args) if inspect.isfunction(check): - args = inspect.getargspec(check)[0] + args = _get_parameters(check) if args and args[0] in ('physical_line', 'logical_line'): if codes is None: codes = ERRORCODE_REGEX.findall(check.__doc__ or '') _add_check(check, args[0], codes, args) elif inspect.isclass(check): - if inspect.getargspec(check.__init__)[0][:2] == ['self', 'tree']: + if _get_parameters(check.__init__)[:2] == ['self', 'tree']: _add_check(check, 'tree', codes, None) @@ -1419,7 +1463,7 @@ class Checker(object): return check(*arguments) def init_checker_state(self, name, argument_names): - """ Prepares a custom state for the specific checker plugin.""" + """Prepare custom state for the specific checker plugin.""" if 'checker_state' in argument_names: self.checker_state = self._checker_states.setdefault(name, {}) @@ -1504,7 +1548,7 @@ class Checker(object): """Build the file's AST and run all AST checks.""" try: tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) - except (SyntaxError, TypeError): + except (ValueError, SyntaxError, TypeError): return self.report_invalid_syntax() for name, cls, __ in self._ast_checks: checker = cls(tree, self.filename) @@ -1701,6 +1745,7 @@ class BaseReport(object): class FileReport(BaseReport): """Collect the results of the checks and print only the filenames.""" + print_filename = True @@ -1908,6 +1953,7 @@ class StyleGuide(object): def get_parser(prog='pep8', version=__version__): + """Create the parser for the program.""" parser = OptionParser(prog=prog, version=version, usage="%prog [options] input ...") parser.config_options = [ @@ -1955,8 +2001,8 @@ def get_parser(prog='pep8', version=__version__): parser.add_option('--format', metavar='format', default='default', help="set the error format [default|pylint|]") parser.add_option('--diff', action='store_true', - help="report only lines changed according to the " - "unified diff received on STDIN") + help="report changes only within line number ranges in " + "the unified diff received on STDIN") group = parser.add_option_group("Testing Options") if os.path.exists(TESTSUITE_PATH): group.add_option('--testsuite', metavar='dir', @@ -1969,7 +2015,7 @@ def get_parser(prog='pep8', version=__version__): def read_config(options, args, arglist, parser): - """Read and parse configurations + """Read and parse configurations. If a config file is specified on the command line with the "--config" option, then only it is used for configuration. @@ -1984,24 +2030,24 @@ def read_config(options, args, arglist, parser): local_dir = os.curdir + if USER_CONFIG and os.path.isfile(USER_CONFIG): + if options.verbose: + print('user configuration: %s' % USER_CONFIG) + config.read(USER_CONFIG) + + parent = tail = args and os.path.abspath(os.path.commonprefix(args)) + while tail: + if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): + local_dir = parent + if options.verbose: + print('local configuration: in %s' % parent) + break + (parent, tail) = os.path.split(parent) + if cli_conf and os.path.isfile(cli_conf): if options.verbose: print('cli configuration: %s' % cli_conf) config.read(cli_conf) - else: - if USER_CONFIG and os.path.isfile(USER_CONFIG): - if options.verbose: - print('user configuration: %s' % USER_CONFIG) - config.read(USER_CONFIG) - - parent = tail = args and os.path.abspath(os.path.commonprefix(args)) - while tail: - if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): - local_dir = parent - if options.verbose: - print('local configuration: in %s' % parent) - break - (parent, tail) = os.path.split(parent) pep8_section = parser.prog if config.has_section(pep8_section): @@ -2074,10 +2120,10 @@ def process_options(arglist=None, parse_argv=False, config_file=None, options = read_config(options, args, arglist, parser) options.reporter = parse_argv and options.quiet == 1 and FileReport - options.filename = options.filename and options.filename.split(',') + options.filename = _parse_multi_options(options.filename) options.exclude = normalize_paths(options.exclude) - options.select = options.select and options.select.split(',') - options.ignore = options.ignore and options.ignore.split(',') + options.select = _parse_multi_options(options.select) + options.ignore = _parse_multi_options(options.ignore) if options.diff: options.reporter = DiffReport @@ -2088,6 +2134,22 @@ def process_options(arglist=None, parse_argv=False, config_file=None, return options, args +def _parse_multi_options(options, split_token=','): + r"""Split and strip and discard empties. + + Turns the following: + + A, + B, + + into ["A", "B"] + """ + if options: + return [o.strip() for o in options.split(split_token) if o.strip()] + else: + return options + + def _main(): """Parse options and run checks on Python source.""" import signal @@ -2098,23 +2160,29 @@ def _main(): except AttributeError: pass # not supported on Windows - pep8style = StyleGuide(parse_argv=True) - options = pep8style.options + style_guide = StyleGuide(parse_argv=True) + options = style_guide.options + if options.doctest or options.testsuite: from testsuite.support import run_tests - report = run_tests(pep8style) + report = run_tests(style_guide) else: - report = pep8style.check_files() + report = style_guide.check_files() + if options.statistics: report.print_statistics() + if options.benchmark: report.print_benchmark() + if options.testsuite and not options.quiet: report.print_results() + if report.total_errors: if options.count: sys.stderr.write(str(report.total_errors) + '\n') sys.exit(1) + if __name__ == '__main__': _main() diff --git a/python/src/com/jetbrains/python/PythonHelper.java b/python/src/com/jetbrains/python/PythonHelper.java index 1eee40c98919..a99091e5f7d1 100644 --- a/python/src/com/jetbrains/python/PythonHelper.java +++ b/python/src/com/jetbrains/python/PythonHelper.java @@ -76,7 +76,7 @@ public enum PythonHelper implements HelperPackage { EXTRA_SYSPATH("extra_syspath.py"), SYSPATH("syspath.py"), - PEP8("pep8.py"), + PYCODESTYLE("pycodestyle.py"), REST_RUNNER("rest_runners/rst2smth.py"), diff --git a/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java b/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java index 5530696d5b2a..6dd625f27047 100644 --- a/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java +++ b/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java @@ -37,7 +37,6 @@ import com.intellij.openapi.editor.ex.EditorSettingsExternalizable; import com.intellij.openapi.module.ModuleUtilCore; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; -import com.intellij.openapi.util.Computable; import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; @@ -169,7 +168,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator allSdks = PythonSdkType.getAllSdks(); Collections.sort(allSdks, PreferredSdkComparator.INSTANCE); for (Sdk sdk : allSdks) { @@ -190,7 +189,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator