From fc8a8bdedbb77d4101f78a78791440b51c185fe3 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 11:43:56 +0100 Subject: [PATCH] pull in latest pep8.py from github --- python/helpers/pep8.py | 686 ++++++++++++++++++----------------------- 1 file changed, 307 insertions(+), 379 deletions(-) diff --git a/python/helpers/pep8.py b/python/helpers/pep8.py index 3539725a3c34..60e85f373b0d 100644 --- a/python/helpers/pep8.py +++ b/python/helpers/pep8.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # pep8.py - Check Python source code formatting, according to PEP 8 -# Copyright (C) 2006 Johann C. Rocholl +# Copyright (C) 2006-2009 Johann C. Rocholl +# Copyright (C) 2009-2013 Florent Xicluna # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation files @@ -43,57 +44,8 @@ W warnings 600 deprecation 700 statements 900 syntax error - -You can add checks to this program by writing plugins. Each plugin is -a simple function that is called for each line of source code, either -physical or logical. - -Physical line: -- Raw line of text from the input file. - -Logical line: -- Multi-line statements converted to a single line. -- Stripped left and right. -- Contents of strings replaced with 'xxx' of same length. -- Comments removed. - -The check function requests physical or logical lines by the name of -the first argument: - -def maximum_line_length(physical_line) -def extraneous_whitespace(logical_line) -def blank_lines(logical_line, blank_lines, indent_level, line_number) - -The last example above demonstrates how check plugins can request -additional information with extra arguments. All attributes of the -Checker object are available. Some examples: - -lines: a list of the raw lines from the input file -tokens: the tokens that contribute to this logical line -line_number: line number in the input file -blank_lines: blank lines before this one -indent_char: first indentation character in this file (' ' or '\t') -indent_level: indentation (with tabs expanded to multiples of 8) -previous_indent_level: indentation on previous line -previous_logical: previous logical line - -The docstring of each check function shall be the relevant part of -text from PEP 8. It is printed if the user enables --show-pep8. -Several docstrings contain examples directly from the PEP 8 document. - -Okay: spam(ham[1], {eggs: 2}) -E201: spam( ham[1], {eggs: 2}) - -These examples are verified automatically when pep8.py is run with the ---doctest option. You can add examples for your own check functions. -The format is simple: "Okay" or error/warning code followed by colon -and space, the rest of the line is example source code. If you put 'r' -before the docstring, you can use \n for newline, \t for tab and \s -for space. - """ - -__version__ = '1.3.4a0' +__version__ = '1.4.5a0' import os import sys @@ -110,28 +62,30 @@ try: except ImportError: from ConfigParser import RawConfigParser -DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git' -DEFAULT_IGNORE = 'E24' +DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__' +DEFAULT_IGNORE = 'E226,E24' if sys.platform == 'win32': DEFAULT_CONFIG = os.path.expanduser(r'~\.pep8') else: DEFAULT_CONFIG = os.path.join(os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), 'pep8') +PROJECT_CONFIG = ('.pep8', 'tox.ini', 'setup.cfg') +TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') MAX_LINE_LENGTH = 79 REPORT_FORMAT = { 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', } - +PyCF_ONLY_AST = 1024 SINGLETONS = frozenset(['False', 'None', 'True']) KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS -BINARY_OPERATORS = frozenset([ - '**=', '*=', '+=', '-=', '!=', '<>', - '%=', '^=', '&=', '|=', '==', '/=', '//=', '<=', '>=', '<<=', '>>=', - '%', '^', '&', '|', '=', '/', '//', '<', '>', '<<']) UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) -OPERATORS = BINARY_OPERATORS | UNARY_OPERATORS +ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) +WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) +WS_NEEDED_OPERATORS = frozenset([ + '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', + '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) WHITESPACE = frozenset(' \t') SKIP_TOKENS = frozenset([tokenize.COMMENT, tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT]) @@ -140,19 +94,17 @@ BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] INDENT_REGEX = re.compile(r'([ \t]*)') RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*(,)') RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,\s*\w+\s*,\s*\w+') -SELFTEST_REGEX = re.compile(r'(Okay|[EW]\d{3}):\s(.*)') -ERRORCODE_REGEX = re.compile(r'[EW]\d{3}') +ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') DOCSTRING_REGEX = re.compile(r'u?r?["\']') EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') COMPARE_SINGLETON_REGEX = re.compile(r'([=!]=)\s*(None|False|True)') -COMPARE_TYPE_REGEX = re.compile(r'([=!]=|is|is\s+not)\s*type(?:s\.(\w+)Type' - r'|\(\s*(\(\s*\)|[^)]*[^ )])\s*\))') -KEYWORD_REGEX = re.compile(r'(?:[^\s])(\s*)\b(?:%s)\b(\s*)' % - r'|'.join(KEYWORDS)) -OPERATOR_REGEX = re.compile(r'(?:[^\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') +COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' + r'|\s*\(\s*([^)]*[^ )])\s*\))') +KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) +OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') LAMBDA_REGEX = re.compile(r'\blambda\b') -HUNK_REGEX = re.compile(r'^@@ -\d+,\d+ \+(\d+),(\d+) @@.*$') +HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') # Work around Python < 2.6 behaviour, which does not generate NL after # a comment which is on a line by itself. @@ -212,8 +164,8 @@ def trailing_whitespace(physical_line): The warning returned varies on whether the line itself is blank, for easier filtering for those who want to indent their blank lines. - Okay: spam(1) - W291: spam(1)\s + Okay: spam(1)\n# + W291: spam(1) \n# W293: class Foo(object):\n \n bang = 12 """ physical_line = physical_line.rstrip('\n') # chr(10), newline @@ -264,6 +216,8 @@ def maximum_line_length(physical_line, max_line_length): line = physical_line.rstrip() length = len(line) if length > max_line_length: + if noqa(line): + return if hasattr(line, 'decode'): # Python 2 # The line could contain multi-byte characters try: @@ -387,13 +341,15 @@ def missing_whitespace(logical_line): Okay: a[1:4:2] E231: ['a','b'] E231: foo(bar,baz) + E231: [{'a':'b'}] """ line = logical_line for index in range(len(line) - 1): char = line[index] if char in ',;:' and line[index + 1] not in WHITESPACE: before = line[:index] - if char == ':' and before.count('[') > before.count(']'): + if char == ':' and before.count('[') > before.count(']') and \ + before.rfind('{') < before.rfind('['): continue # Slice syntax, no space required if char == ',' and line[index + 1] == ')': continue # Allow tuple with only one element: (3,) @@ -455,7 +411,7 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): """ first_row = tokens[0][2][0] nrows = 1 + tokens[-1][2][0] - first_row - if nrows == 1: + if nrows == 1 or noqa(tokens[0][4]): return # indent_next tells us whether the next block is indented; assuming @@ -470,13 +426,14 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): # relative indents of physical lines rel_indent = [0] * nrows # visual indents - indent = [indent_level] indent_chances = {} last_indent = tokens[0][2] + indent = [last_indent[1]] if verbose >= 3: print(">>> " + tokens[0][4].rstrip()) for token_type, text, start, end, line in tokens: + newline = row < start[0] - first_row if newline: row = start[0] - first_row @@ -490,7 +447,7 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): print("... " + line.rstrip()) # record the initial indent. - rel_indent[row] = start[1] - indent_level + rel_indent[row] = expand_indent(line) - indent_level if depth: # a bracket expression in a continuation line. @@ -508,11 +465,11 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): # this line starts with a closing bracket if indent[depth]: if start[1] != indent[depth]: - yield (start, 'E124 closing bracket does not match ' - 'visual indentation') + yield (start, "E124 closing bracket does not match " + "visual indentation") elif hang: - yield (start, 'E123 closing bracket does not match ' - 'indentation of opening bracket\'s line') + yield (start, "E123 closing bracket does not match " + "indentation of opening bracket's line") elif visual_indent is True: # visual indent is verified if not indent[depth]: @@ -522,21 +479,21 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): pass elif indent[depth] and start[1] < indent[depth]: # visual indent is broken - yield (start, 'E128 continuation line ' - 'under-indented for visual indent') + yield (start, "E128 continuation line " + "under-indented for visual indent") elif hang == 4 or (indent_next and rel_indent[row] == 8): # hanging indent is verified pass else: # indent is broken if hang <= 0: - error = 'E122', 'missing indentation or outdented' + error = "E122", "missing indentation or outdented" elif indent[depth]: - error = 'E127', 'over-indented for visual indent' + error = "E127", "over-indented for visual indent" elif hang % 4: - error = 'E121', 'indentation is not a multiple of four' + error = "E121", "indentation is not a multiple of four" else: - error = 'E126', 'over-indented for hanging indent' + error = "E126", "over-indented for hanging indent" yield start, "%s continuation line %s" % error # look for visual indenting @@ -547,8 +504,12 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): if verbose >= 4: print("bracket depth %s indent to %s" % (depth, start[1])) # deal with implicit string concatenation - elif token_type == tokenize.STRING or text in ('u', 'ur', 'b', 'br'): + elif (token_type in (tokenize.STRING, tokenize.COMMENT) or + text in ('u', 'ur', 'b', 'br')): indent_chances[start[1]] = str + # special case for the "if" statement because len("if (") == 4 + elif not indent_chances and not row and not depth and text == 'if': + indent_chances[end[1] + 1] = True # keep track of bracket depth if token_type == tokenize.OP: @@ -665,20 +626,16 @@ def missing_whitespace_around_operator(logical_line, tokens): Okay: hypot2 = x * x + y * y Okay: c = (a + b) * (a - b) Okay: foo(bar, key='word', *args, **kwargs) - Okay: baz(**kwargs) - Okay: negative = -1 - Okay: spam(-1) Okay: alpha[:-i] - Okay: if not -5 < x < +5:\n pass - Okay: lambda *args, **kw: (args, kw) E225: i=i+1 E225: submitted +=1 - E225: x = x*2 - 1 - E225: hypot2 = x*x + y*y - E225: c = (a+b) * (a-b) - E225: c = alpha -4 + E225: x = x /2 - 1 E225: z = x **y + E226: c = (a+b) * (a-b) + E226: hypot2 = x*x + y*y + E227: c = a|b + E228: msg = fmt%(errno, errmsg) """ parens = 0 need_space = False @@ -686,7 +643,7 @@ def missing_whitespace_around_operator(logical_line, tokens): prev_text = prev_end = None for token_type, text, start, end, line in tokens: if token_type in (tokenize.NL, tokenize.NEWLINE, tokenize.ERRORTOKEN): - # ERRORTOKEN is triggered by backticks in Python 3000 + # ERRORTOKEN is triggered by backticks in Python 3 continue if text in ('(', 'lambda'): parens += 1 @@ -694,32 +651,59 @@ def missing_whitespace_around_operator(logical_line, tokens): parens -= 1 if need_space: if start != prev_end: + # Found a (probably) needed space + if need_space is not True and not need_space[1]: + yield (need_space[0], + "E225 missing whitespace around operator") need_space = False elif text == '>' and prev_text in ('<', '-'): # Tolerate the "<>" operator, even if running Python 3 # Deal with Python 3's annotated return value "->" pass else: - yield prev_end, "E225 missing whitespace around operator" + if need_space is True or need_space[1]: + # A needed trailing space was not found + yield prev_end, "E225 missing whitespace around operator" + else: + code, optype = 'E226', 'arithmetic' + if prev_text == '%': + code, optype = 'E228', 'modulo' + elif prev_text not in ARITHMETIC_OP: + code, optype = 'E227', 'bitwise or shift' + yield (need_space[0], "%s missing whitespace " + "around %s operator" % (code, optype)) need_space = False elif token_type == tokenize.OP and prev_end is not None: if text == '=' and parens: # Allow keyword args or defaults: foo(bar=None). pass - elif text in BINARY_OPERATORS: + elif text in WS_NEEDED_OPERATORS: need_space = True elif text in UNARY_OPERATORS: + # Check if the operator is being used as a binary operator # Allow unary operators: -123, -x, +1. # Allow argument unpacking: foo(*args, **kwargs). if prev_type == tokenize.OP: - if prev_text in '}])': - need_space = True + binary_usage = (prev_text in '}])') elif prev_type == tokenize.NAME: - if prev_text not in KEYWORDS: + binary_usage = (prev_text not in KEYWORDS) + else: + binary_usage = (prev_type not in SKIP_TOKENS) + + if binary_usage: + if text in WS_OPTIONAL_OPERATORS: + need_space = None + else: need_space = True - elif prev_type not in SKIP_TOKENS: - need_space = True - if need_space and start == prev_end: + elif text in WS_OPTIONAL_OPERATORS: + need_space = None + + if need_space is None: + # Surrounding space is optional, but ensure that + # trailing space matches opening space + need_space = (prev_end, start != prev_end) + elif need_space and start == prev_end: + # A needed opening space was not found yield prev_end, "E225 missing whitespace around operator" need_space = False prev_type = token_type @@ -767,12 +751,12 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): parens = 0 no_space = False prev_end = None + message = "E251 unexpected spaces around keyword / parameter equals" for token_type, text, start, end, line in tokens: if no_space: no_space = False if start != prev_end: - yield (prev_end, - "E251 no spaces around keyword / parameter equals") + yield (prev_end, message) elif token_type == tokenize.OP: if text == '(': parens += 1 @@ -781,8 +765,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): elif parens and text == '=': no_space = True if start != prev_end: - yield (prev_end, - "E251 no spaces around keyword / parameter equals") + yield (prev_end, message) prev_end = end @@ -808,7 +791,8 @@ def whitespace_before_inline_comment(logical_line, tokens): if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: yield (prev_end, "E261 at least two spaces before inline comment") - if text.startswith('# ') or not text.startswith('# '): + symbol, sp, comment = text.partition(' ') + if symbol not in ('#', '#:') or comment[:1].isspace(): yield start, "E262 inline comment should start with '# '" elif token_type != tokenize.NL: prev_end = end @@ -858,10 +842,12 @@ def compound_statements(logical_line): E701: if foo == 'blah': one(); two(); three() E702: do_one(); do_two(); do_three() + E703: do_four(); # useless semicolon """ line = logical_line + last_char = len(line) - 1 found = line.find(':') - if -1 < found < len(line) - 1: + if -1 < found < last_char: before = line[:found] if (before.count('{') <= before.count('}') and # {'a': 1} (dict) before.count('[') <= before.count(']') and # [1:2] (slice) @@ -870,7 +856,10 @@ def compound_statements(logical_line): yield found, "E701 multiple statements on one line (colon)" found = line.find(';') if -1 < found: - yield found, "E702 multiple statements on one line (semicolon)" + if found < last_char: + yield found, "E702 multiple statements on one line (semicolon)" + else: + yield found, "E703 statement ends with a semicolon" def explicit_line_join(logical_line, tokens): @@ -955,16 +944,16 @@ def comparison_type(logical_line): """ match = COMPARE_TYPE_REGEX.search(logical_line) if match: - inst = match.group(3) + inst = match.group(1) if inst and isidentifier(inst) and inst not in SINGLETONS: return # Allow comparison for types which are not obvious - yield match.start(1), "E721 do not compare types, use 'isinstance()'" + yield match.start(0), "E721 do not compare types, use 'isinstance()'" def python_3000_has_key(logical_line): r""" - The {}.has_key() method will be removed in the future version of - Python. Use the 'in' operation instead. + The {}.has_key() method is removed in the Python 3. + Use the 'in' operation instead. Okay: if "alph" in d:\n print d["alph"] W601: assert d.has_key('alph') @@ -982,7 +971,7 @@ def python_3000_raise_comma(logical_line): The paren-using form is preferred because when the exception arguments are long or include string formatting, you don't need to use line continuation characters thanks to the containing parentheses. The older - form will be removed in Python 3000. + form is removed in Python 3. Okay: raise DummyError("Message") W602: raise DummyError, "Message" @@ -996,7 +985,7 @@ def python_3000_not_equal(logical_line): """ != can also be written <>, but this is an obsolete usage kept for backwards compatibility only. New code should always use !=. - The older syntax is removed in Python 3000. + The older syntax is removed in Python 3. Okay: if a != 'no': W603: if a <> 'no': @@ -1008,7 +997,7 @@ def python_3000_not_equal(logical_line): def python_3000_backticks(logical_line): """ - Backticks are removed in Python 3000. + Backticks are removed in Python 3. Use repr() instead. Okay: val = repr(1 + 2) @@ -1056,6 +1045,7 @@ else: def stdin_get_value(): return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() readlines.__doc__ = " Read the source code." +noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search def expand_indent(line): @@ -1109,6 +1099,9 @@ def mute_string(text): def parse_udiff(diff, patterns=None, parent='.'): + """Return a dictionary of matching lines.""" + # For each file of the diff, the entry key is the filename, + # and the value is a set of row numbers to consider. rv = {} path = nrows = None for line in diff.splitlines(): @@ -1117,7 +1110,8 @@ def parse_udiff(diff, patterns=None, parent='.'): nrows -= 1 continue if line[:3] == '@@ ': - row, nrows = [int(g) for g in HUNK_REGEX.match(line).groups()] + hunk_match = HUNK_REGEX.match(line) + row, nrows = [int(g or '1') for g in hunk_match.groups()] rv[path].update(range(row, row + nrows)) elif line[:3] == '+++': path = line[4:].split('\t', 1)[0] @@ -1144,18 +1138,38 @@ def filename_match(filename, patterns, default=True): ############################################################################## -def find_checks(argument_name): +_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} + + +def register_check(check, codes=None): """ - Find all globally visible functions where the first argument name - starts with argument_name. + Register a new check object. """ - for name, function in globals().items(): - if not inspect.isfunction(function): - continue - args = inspect.getargspec(function)[0] - if args and args[0].startswith(argument_name): - codes = ERRORCODE_REGEX.findall(function.__doc__ or '') - yield name, codes, function, args + def _add_check(check, kind, codes, args): + if check in _checks[kind]: + _checks[kind][check][0].extend(codes or []) + else: + _checks[kind][check] = (codes or [''], args) + if inspect.isfunction(check): + args = inspect.getargspec(check)[0] + if args and args[0] in ('physical_line', 'logical_line'): + if codes is None: + codes = ERRORCODE_REGEX.findall(check.__doc__ or '') + _add_check(check, args[0], codes, args) + elif inspect.isclass(check): + if inspect.getargspec(check.__init__)[0][:2] == ['self', 'tree']: + _add_check(check, 'tree', codes, None) + + +def init_checks_registry(): + """ + Register all globally visible functions where the first argument name + is 'physical_line' or 'logical_line'. + """ + mod = inspect.getmodule(register_check) + for (name, function) in inspect.getmembers(mod, inspect.isfunction): + register_check(function) +init_checks_registry() class Checker(object): @@ -1163,7 +1177,7 @@ class Checker(object): Load a Python source file, tokenize it, check coding style. """ - def __init__(self, filename, lines=None, + def __init__(self, filename=None, lines=None, options=None, report=None, **kwargs): if options is None: options = StyleGuide(kwargs).options @@ -1172,6 +1186,7 @@ class Checker(object): self._io_error = None self._physical_checks = options.physical_checks self._logical_checks = options.logical_checks + self._ast_checks = options.ast_checks self.max_line_length = options.max_line_length self.verbose = options.verbose self.filename = filename @@ -1193,6 +1208,16 @@ class Checker(object): self.report = report or options.report self.report_error = self.report.error + def report_invalid_syntax(self): + exc_type, exc = sys.exc_info()[:2] + offset = exc.args[1] + if len(offset) > 2: + offset = offset[1:3] + self.report_error(offset[0], offset[1] or 0, + 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), + self.report_invalid_syntax) + report_invalid_syntax.__doc__ = " Check if the syntax is valid." + def readline(self): """ Get the next line from the input buffer. @@ -1266,7 +1291,8 @@ class Checker(object): length += len(text) previous = token self.logical_line = ''.join(logical) - assert self.logical_line.strip() == self.logical_line + # With Python 2, if the line ends with '\r\r\n' the assertion fails + # assert self.logical_line.strip() == self.logical_line def check_logical(self): """ @@ -1295,6 +1321,17 @@ class Checker(object): self.report_error(orig_number, orig_offset, text, check) self.previous_logical = self.logical_line + def check_ast(self): + try: + tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + except SyntaxError: + return self.report_invalid_syntax() + for name, cls, _ in self._ast_checks: + checker = cls(tree, self.filename) + for lineno, offset, text, check in checker.run(): + if not noqa(self.lines[lineno - 1]): + self.report_error(lineno, offset, text, check) + def generate_tokens(self): if self._io_error: self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) @@ -1303,20 +1340,15 @@ class Checker(object): for token in tokengen: yield token except (SyntaxError, tokenize.TokenError): - exc_type, exc = sys.exc_info()[:2] - offset = exc.args[1] - if len(offset) > 2: - offset = offset[1:3] - self.report_error(offset[0], offset[1], - 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), - self.generate_tokens) - generate_tokens.__doc__ = " Check if the syntax is valid." + self.report_invalid_syntax() def check_all(self, expected=None, line_offset=0): """ Run all checks on the input file. """ self.report.init_file(self.filename, self.lines, expected, line_offset) + if self._ast_checks: + self.check_ast() self.line_number = 0 self.indent_char = None self.indent_level = 0 @@ -1452,6 +1484,7 @@ class BaseReport(object): class FileReport(BaseReport): + """Collect the results of the checks and print only the filenames.""" print_filename = True @@ -1466,17 +1499,29 @@ class StandardReport(BaseReport): self._show_source = options.show_source self._show_pep8 = options.show_pep8 + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self._deferred_print = [] + return super(StandardReport, self).init_file( + filename, lines, expected, line_offset) + def error(self, line_number, offset, text, check): - """ - Report an error, according to options. - """ + """Report an error, according to options.""" code = super(StandardReport, self).error(line_number, offset, text, check) if code and (self.counters[code] == 1 or self._repeat): + self._deferred_print.append( + (line_number, offset, code, text[5:], check.__doc__)) + return code + + def get_file_results(self): + """Print the result and return the overall count for this file.""" + self._deferred_print.sort() + for line_number, offset, code, text, doc in self._deferred_print: print(self._fmt % { 'path': self.filename, 'row': self.line_offset + line_number, 'col': offset + 1, - 'code': code, 'text': text[5:], + 'code': code, 'text': text, }) if self._show_source: if line_number > len(self.lines): @@ -1485,9 +1530,9 @@ class StandardReport(BaseReport): line = self.lines[line_number - 1] print(line.rstrip()) print(' ' * offset + '^') - if self._show_pep8: - print(check.__doc__.lstrip('\n').rstrip()) - return code + if self._show_pep8 and doc: + print(doc.lstrip('\n').rstrip()) + return self.file_errors class DiffReport(StandardReport): @@ -1503,54 +1548,17 @@ class DiffReport(StandardReport): return super(DiffReport, self).error(line_number, offset, text, check) -class TestReport(StandardReport): - """Collect the results for the tests.""" - - def __init__(self, options): - options.benchmark_keys += ['test cases', 'failed tests'] - super(TestReport, self).__init__(options) - self._verbose = options.verbose - - def get_file_results(self): - # Check if the expected errors were found - label = '%s:%s:1' % (self.filename, self.line_offset) - codes = sorted(self.expected) - for code in codes: - if not self.counters.get(code): - self.file_errors += 1 - self.total_errors += 1 - print('%s: error %s not found' % (label, code)) - if self._verbose and not self.file_errors: - print('%s: passed (%s)' % - (label, ' '.join(codes) or 'Okay')) - self.counters['test cases'] += 1 - if self.file_errors: - self.counters['failed tests'] += 1 - # Reset counters - for key in set(self.counters) - set(self._benchmark_keys): - del self.counters[key] - self.messages = {} - return self.file_errors - - def print_results(self): - results = ("%(physical lines)d lines tested: %(files)d files, " - "%(test cases)d test cases%%s." % self.counters) - if self.total_errors: - print(results % ", %s failures" % self.total_errors) - else: - print(results % "") - print("Test failed." if self.total_errors else "Test passed.") - - class StyleGuide(object): """Initialize a PEP-8 instance with few options.""" def __init__(self, *args, **kwargs): # build options from the command line + self.checker_class = kwargs.pop('checker_class', Checker) parse_argv = kwargs.pop('parse_argv', False) config_file = kwargs.pop('config_file', None) - options, self.paths = process_options(parse_argv=parse_argv, - config_file=config_file) + parser = kwargs.pop('parser', None) + options, self.paths = process_options( + parse_argv=parse_argv, config_file=config_file, parser=parser) if args or kwargs: # build options from dict options_dict = dict(*args, **kwargs) @@ -1573,6 +1581,7 @@ class StyleGuide(object): options.ignore_code = self.ignore_code options.physical_checks = self.get_checks('physical_line') options.logical_checks = self.get_checks('logical_line') + options.ast_checks = self.get_checks('tree') self.init_report() def init_report(self, reporter=None): @@ -1587,11 +1596,14 @@ class StyleGuide(object): report = self.options.report runner = self.runner report.start() - for path in paths: - if os.path.isdir(path): - self.input_dir(path) - elif not self.excluded(path): - runner(path) + try: + for path in paths: + if os.path.isdir(path): + self.input_dir(path) + elif not self.excluded(path): + runner(path) + except KeyboardInterrupt: + print('... stopped') report.stop() return report @@ -1599,7 +1611,8 @@ class StyleGuide(object): """Run all checks on a Python source file.""" if self.options.verbose: print('checking %s' % filename) - fchecker = Checker(filename, lines=lines, options=self.options) + fchecker = self.checker_class( + filename, lines=lines, options=self.options) return fchecker.check_all(expected=expected, line_offset=line_offset) def input_dir(self, dirname): @@ -1616,7 +1629,7 @@ class StyleGuide(object): print('directory ' + root) counters['directories'] += 1 for subdir in sorted(dirs): - if self.excluded(subdir): + if self.excluded(os.path.join(root, subdir)): dirs.remove(subdir) for filename in sorted(files): # contain a pattern that matches? @@ -1629,7 +1642,10 @@ class StyleGuide(object): Check if options.exclude contains a pattern that matches filename. """ basename = os.path.basename(filename) - return filename_match(basename, self.options.exclude, default=False) + return any((filename_match(filename, self.options.exclude, + default=False), + filename_match(basename, self.options.exclude, + default=False))) def ignore_code(self, code): """ @@ -1648,168 +1664,15 @@ class StyleGuide(object): starts with argument_name and which contain selected tests. """ checks = [] - for name, codes, function, args in find_checks(argument_name): + for check, attrs in _checks[argument_name].items(): + (codes, args) = attrs if any(not (code and self.ignore_code(code)) for code in codes): - checks.append((name, function, args)) + checks.append((check.__name__, check, args)) return sorted(checks) -def init_tests(pep8style): - """ - Initialize testing framework. - - A test file can provide many tests. Each test starts with a - declaration. This declaration is a single line starting with '#:'. - It declares codes of expected failures, separated by spaces or 'Okay' - if no failure is expected. - If the file does not contain such declaration, it should pass all - tests. If the declaration is empty, following lines are not checked, - until next declaration. - - Examples: - - * Only E224 and W701 are expected: #: E224 W701 - * Following example is conform: #: Okay - * Don't check these lines: #: - """ - report = pep8style.init_report(TestReport) - runner = pep8style.input_file - - def run_tests(filename): - """Run all the tests from a file.""" - lines = readlines(filename) + ['#:\n'] - line_offset = 0 - codes = ['Okay'] - testcase = [] - count_files = report.counters['files'] - for index, line in enumerate(lines): - if not line.startswith('#:'): - if codes: - # Collect the lines of the test case - testcase.append(line) - continue - if codes and index: - codes = [c for c in codes if c != 'Okay'] - # Run the checker - runner(filename, testcase, expected=codes, - line_offset=line_offset) - # output the real line numbers - line_offset = index + 1 - # configure the expected errors - codes = line.split()[1:] - # empty the test case buffer - del testcase[:] - report.counters['files'] = count_files + 1 - return report.counters['failed tests'] - - pep8style.runner = run_tests - - -def selftest(options): - """ - Test all check functions with test cases in docstrings. - """ - count_failed = count_all = 0 - report = BaseReport(options) - counters = report.counters - checks = options.physical_checks + options.logical_checks - for name, check, argument_names in checks: - for line in check.__doc__.splitlines(): - line = line.lstrip() - match = SELFTEST_REGEX.match(line) - if match is None: - continue - code, source = match.groups() - checker = Checker(None, options=options, report=report) - for part in source.split(r'\n'): - part = part.replace(r'\t', '\t') - part = part.replace(r'\s', ' ') - checker.lines.append(part + '\n') - checker.check_all() - error = None - if code == 'Okay': - if len(counters) > len(options.benchmark_keys): - codes = [key for key in counters - if key not in options.benchmark_keys] - error = "incorrectly found %s" % ', '.join(codes) - elif not counters.get(code): - error = "failed to find %s" % code - # Keep showing errors for multiple tests - for key in set(counters) - set(options.benchmark_keys): - del counters[key] - report.messages = {} - count_all += 1 - if not error: - if options.verbose: - print("%s: %s" % (code, source)) - else: - count_failed += 1 - print("%s: %s:" % (__file__, error)) - for line in checker.lines: - print(line.rstrip()) - return count_failed, count_all - - -def read_config(options, args, arglist, parser): - """Read both user configuration and local configuration.""" - config = RawConfigParser() - - user_conf = options.config - if user_conf and os.path.isfile(user_conf): - if options.verbose: - print('user configuration: %s' % user_conf) - config.read(user_conf) - - parent = tail = args and os.path.abspath(os.path.commonprefix(args)) - while tail: - local_conf = os.path.join(parent, '.pep8') - if os.path.isfile(local_conf): - if options.verbose: - print('local configuration: %s' % local_conf) - config.read(local_conf) - break - parent, tail = os.path.split(parent) - - if config.has_section('pep8'): - option_list = dict([(o.dest, o.type or o.action) - for o in parser.option_list]) - - # First, read the default values - new_options, _ = parser.parse_args([]) - - # Second, parse the configuration - for opt in config.options('pep8'): - if options.verbose > 1: - print(' %s = %s' % (opt, config.get('pep8', opt))) - if opt.replace('_', '-') not in parser.config_options: - print('Unknown option: \'%s\'\n not in [%s]' % - (opt, ' '.join(parser.config_options))) - sys.exit(1) - normalized_opt = opt.replace('-', '_') - opt_type = option_list[normalized_opt] - if opt_type in ('int', 'count'): - value = config.getint('pep8', opt) - elif opt_type == 'string': - value = config.get('pep8', opt) - else: - assert opt_type in ('store_true', 'store_false') - value = config.getboolean('pep8', opt) - setattr(new_options, normalized_opt, value) - - # Third, overwrite with the command-line options - options, _ = parser.parse_args(arglist, values=new_options) - - return options - - -def process_options(arglist=None, parse_argv=False, config_file=None): - """Process options passed either via arglist or via command line args.""" - if not arglist and not parse_argv: - # Don't read the command line if the module is used as a library. - arglist = [] - if config_file is True: - config_file = DEFAULT_CONFIG - parser = OptionParser(version=__version__, +def get_parser(prog='pep8', version=__version__): + parser = OptionParser(prog=prog, version=version, usage="%prog [options] input ...") parser.config_options = [ 'exclude', 'filename', 'select', 'ignore', 'max-line-length', 'count', @@ -1854,27 +1717,100 @@ def process_options(arglist=None, parse_argv=False, config_file=None): help="report only lines changed according to the " "unified diff received on STDIN") group = parser.add_option_group("Testing Options") - group.add_option('--testsuite', metavar='dir', - help="run regression tests from dir") - group.add_option('--doctest', action='store_true', - help="run doctest on myself") + if os.path.exists(TESTSUITE_PATH): + group.add_option('--testsuite', metavar='dir', + help="run regression tests from dir") + group.add_option('--doctest', action='store_true', + help="run doctest on myself") group.add_option('--benchmark', action='store_true', help="measure processing speed") - group = parser.add_option_group("Configuration", description=( - "The project options are read from the [pep8] section of the .pep8 " - "file located in any parent folder of the path(s) being processed. " - "Allowed options are: %s." % ', '.join(parser.config_options))) - group.add_option('--config', metavar='path', default=config_file, - help="config file location (default: %default)") + return parser + +def read_config(options, args, arglist, parser): + """Read both user configuration and local configuration.""" + config = RawConfigParser() + + user_conf = options.config + if user_conf and os.path.isfile(user_conf): + if options.verbose: + print('user configuration: %s' % user_conf) + config.read(user_conf) + + parent = tail = args and os.path.abspath(os.path.commonprefix(args)) + while tail: + for name in PROJECT_CONFIG: + local_conf = os.path.join(parent, name) + if os.path.isfile(local_conf): + break + else: + parent, tail = os.path.split(parent) + continue + if options.verbose: + print('local configuration: %s' % local_conf) + config.read(local_conf) + break + + pep8_section = parser.prog + if config.has_section(pep8_section): + option_list = dict([(o.dest, o.type or o.action) + for o in parser.option_list]) + + # First, read the default values + new_options, _ = parser.parse_args([]) + + # Second, parse the configuration + for opt in config.options(pep8_section): + if options.verbose > 1: + print(" %s = %s" % (opt, config.get(pep8_section, opt))) + if opt.replace('_', '-') not in parser.config_options: + print("Unknown option: '%s'\n not in [%s]" % + (opt, ' '.join(parser.config_options))) + sys.exit(1) + normalized_opt = opt.replace('-', '_') + opt_type = option_list[normalized_opt] + if opt_type in ('int', 'count'): + value = config.getint(pep8_section, opt) + elif opt_type == 'string': + value = config.get(pep8_section, opt) + else: + assert opt_type in ('store_true', 'store_false') + value = config.getboolean(pep8_section, opt) + setattr(new_options, normalized_opt, value) + + # Third, overwrite with the command-line options + options, _ = parser.parse_args(arglist, values=new_options) + options.doctest = options.testsuite = False + return options + + +def process_options(arglist=None, parse_argv=False, config_file=None, + parser=None): + """Process options passed either via arglist or via command line args.""" + if not arglist and not parse_argv: + # Don't read the command line if the module is used as a library. + arglist = [] + if not parser: + parser = get_parser() + if not parser.has_option('--config'): + if config_file is True: + config_file = DEFAULT_CONFIG + group = parser.add_option_group("Configuration", description=( + "The project options are read from the [%s] section of the " + "tox.ini file or the setup.cfg file located in any parent folder " + "of the path(s) being processed. Allowed options are: %s." % + (parser.prog, ', '.join(parser.config_options)))) + group.add_option('--config', metavar='path', default=config_file, + help="user config file location (default: %default)") options, args = parser.parse_args(arglist) options.reporter = None - if options.testsuite: + if options.ensure_value('testsuite', False): args.append(options.testsuite) - elif not options.doctest: + elif not options.ensure_value('doctest', False): if parse_argv and not args: - if os.path.exists('.pep8') or options.diff: + if options.diff or any(os.path.exists(name) + for name in PROJECT_CONFIG): args = ['.'] else: parser.error('input not specified') @@ -1907,20 +1843,13 @@ def _main(): """Parse options and run checks on Python source.""" pep8style = StyleGuide(parse_argv=True, config_file=True) options = pep8style.options - if options.doctest: - import doctest - fail_d, done_d = doctest.testmod(report=False, verbose=options.verbose) - fail_s, done_s = selftest(options) - count_failed = fail_s + fail_d - if not options.quiet: - count_passed = done_d + done_s - count_failed - print("%d passed and %d failed." % (count_passed, count_failed)) - print("Test failed." if count_failed else "Test passed.") - if count_failed: - sys.exit(1) - if options.testsuite: - init_tests(pep8style) - report = pep8style.check_files() + if options.doctest or options.testsuite: + sys.path[:0] = [TESTSUITE_PATH] + from test_pep8 import run_tests + del sys.path[0] + report = run_tests(pep8style, options.doctest, options.testsuite) + else: + report = pep8style.check_files() if options.statistics: report.print_statistics() if options.benchmark: @@ -1932,6 +1861,5 @@ def _main(): sys.stderr.write(str(report.total_errors) + '\n') sys.exit(1) - if __name__ == '__main__': _main()