diff --git a/python/helpers/pycodestyle.py b/python/helpers/pycodestyle.py old mode 100755 new mode 100644 index 37a4f3917208..868e79d5a1c5 --- a/python/helpers/pycodestyle.py +++ b/python/helpers/pycodestyle.py @@ -59,16 +59,10 @@ import tokenize import warnings from fnmatch import fnmatch from functools import lru_cache +from itertools import pairwise from optparse import OptionParser -# this is a performance hack. see https://bugs.python.org/issue43014 -if ( - sys.version_info < (3, 10) and - callable(getattr(tokenize, '_compile', None)) -): # pragma: no cover (%&^]+|:=)(\s*)') LAMBDA_REGEX = re.compile(r'\blambda\b') HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') STARTSWITH_DEF_REGEX = re.compile(r'^(async\s+def|def)\b') +STARTSWITH_GENERIC_REGEX = re.compile(r'^(async\s+def|def|class|type)\s+\w+\[') STARTSWITH_TOP_LEVEL_REGEX = re.compile(r'^(async\s+def\s+|def\s+|class\s+|@)') STARTSWITH_INDENT_STATEMENT_REGEX = re.compile( r'^\s*({})\b'.format('|'.join(s.replace(' ', r'\s+') for s in ( @@ -156,6 +152,13 @@ if sys.version_info >= (3, 12): # pragma: >=3.12 cover else: # pragma: <3.12 cover FSTRING_START = FSTRING_MIDDLE = FSTRING_END = -1 +if sys.version_info >= (3, 14): # pragma: >=3.14 cover + TSTRING_START = tokenize.TSTRING_START + TSTRING_MIDDLE = tokenize.TSTRING_MIDDLE + TSTRING_END = tokenize.TSTRING_END +else: # pragma: <3.14 cover + TSTRING_START = TSTRING_MIDDLE = TSTRING_END = -1 + _checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} @@ -232,9 +235,11 @@ def trailing_whitespace(physical_line): W291: spam(1) \n# W293: class Foo(object):\n \n bang = 12 """ - physical_line = physical_line.rstrip('\n') # chr(10), newline - physical_line = physical_line.rstrip('\r') # chr(13), carriage return - physical_line = physical_line.rstrip('\x0c') # chr(12), form feed, ^L + # Strip these trailing characters: + # - chr(10), newline + # - chr(13), carriage return + # - chr(12), form feed, ^L + physical_line = physical_line.rstrip('\n\r\x0c') stripped = physical_line.rstrip(' \t\v') if physical_line != stripped: if stripped: @@ -438,6 +443,9 @@ def extraneous_whitespace(logical_line): E203: if x == 4: print x, y; x, y = y , x E203: if x == 4: print x, y ; x, y = y, x E203: if x == 4 : print x, y; x, y = y, x + + Okay: @decorator + E204: @ decorator """ line = logical_line for match in EXTRANEOUS_WHITESPACE_REGEX.finditer(line): @@ -451,6 +459,9 @@ def extraneous_whitespace(logical_line): code = ('E202' if char in '}])' else 'E203') # if char in ',;:' yield found, f"{code} whitespace before '{char}'" + if WHITESPACE_AFTER_DECORATOR_REGEX.match(logical_line): + yield 1, "E204 whitespace after decorator '@'" + @register_check def whitespace_around_keywords(logical_line): @@ -485,16 +496,17 @@ def missing_whitespace_after_keyword(logical_line, tokens): E275: from importable.module import(bar, baz) E275: if(foo): bar """ - for tok0, tok1 in zip(tokens, tokens[1:]): + for tok0, tok1 in pairwise(tokens): # This must exclude the True/False/None singletons, which can # appear e.g. as "if x is None:", and async/await, which were # valid identifier names in old Python versions. if (tok0.end == tok1.start and + tok0.type == tokenize.NAME and keyword.iskeyword(tok0.string) and tok0.string not in SINGLETONS and not (tok0.string == 'except' and tok1.string == '*') and not (tok0.string == 'yield' and tok1.string == ')') and - tok1.string not in ':\n'): + (tok1.string and tok1.string != ':' and tok1.string != '\n')): yield tok0.end, "E275 missing whitespace after keyword" @@ -686,7 +698,12 @@ def continued_indentation(logical_line, tokens, indent_level, hang_closing, if verbose >= 4: print(f"bracket depth {depth} indent to {start[1]}") # deal with implicit string concatenation - elif token_type in (tokenize.STRING, tokenize.COMMENT, FSTRING_START): + elif token_type in { + tokenize.STRING, + tokenize.COMMENT, + FSTRING_START, + TSTRING_START + }: indent_chances[start[1]] = str # visual indent after assert/raise/with elif not row and not depth and text in ["assert", "raise", "with"]: @@ -775,7 +792,6 @@ def whitespace_before_parameters(logical_line, tokens): # Allow "return (a.foo for a in range(5))" not keyword.iskeyword(prev_text) and ( - sys.version_info < (3, 9) or # 3.12+: type is a soft keyword but no braces after prev_text == 'type' or not keyword.issoftkeyword(prev_text) @@ -863,6 +879,8 @@ def missing_whitespace(logical_line, tokens): brace_stack.append(text) elif token_type == FSTRING_START: # pragma: >=3.12 cover brace_stack.append('f') + elif token_type == TSTRING_START: # pragma: >=3.14 cover + brace_stack.append('t') elif token_type == tokenize.NAME and text == 'lambda': brace_stack.append('l') elif brace_stack: @@ -870,6 +888,8 @@ def missing_whitespace(logical_line, tokens): brace_stack.pop() elif token_type == FSTRING_END: # pragma: >=3.12 cover brace_stack.pop() + elif token_type == TSTRING_END: # pragma: >=3.14 cover + brace_stack.pop() elif ( brace_stack[-1] == 'l' and token_type == tokenize.OP and @@ -889,6 +909,9 @@ def missing_whitespace(logical_line, tokens): # 3.12+ fstring format specifier elif text == ':' and brace_stack[-2:] == ['f', '{']: # pragma: >=3.12 cover # noqa: E501 pass + # 3.14+ tstring format specifier + elif text == ':' and brace_stack[-2:] == ['t', '{']: # pragma: >=3.14 cover # noqa: E501 + pass # tuple (and list for some reason?) elif text == ',' and next_char in ')]': pass @@ -938,7 +961,9 @@ def missing_whitespace(logical_line, tokens): # allow keyword args or defaults: foo(bar=None). brace_stack[-1:] == ['('] or # allow python 3.8 fstring repr specifier - brace_stack[-2:] == ['f', '{'] + brace_stack[-2:] == ['f', '{'] or + # allow python 3.8 fstring repr specifier + brace_stack[-2:] == ['t', '{'] ) ): pass @@ -950,10 +975,8 @@ def missing_whitespace(logical_line, tokens): # Allow argument unpacking: foo(*args, **kwargs). if prev_type == tokenize.OP and prev_text in '}])' or ( prev_type != tokenize.OP and - prev_text not in KEYWORDS and ( - sys.version_info < (3, 9) or - not keyword.issoftkeyword(prev_text) - ) + prev_text not in KEYWORDS and + not keyword.issoftkeyword(prev_text) ): need_space = None elif text in WS_OPTIONAL_OPERATORS: @@ -1012,12 +1035,13 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): E251: return magic(r = real, i = imag) E252: def complex(real, image: float=0.0): """ - parens = 0 + paren_stack = [] no_space = False require_space = False prev_end = None annotated_func_arg = False in_def = bool(STARTSWITH_DEF_REGEX.match(logical_line)) + in_generic = bool(STARTSWITH_GENERIC_REGEX.match(logical_line)) message = "E251 unexpected spaces around keyword / parameter equals" missing_message = "E252 missing whitespace around parameter equals" @@ -1035,15 +1059,23 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): yield (prev_end, missing_message) if token_type == tokenize.OP: if text in '([': - parens += 1 - elif text in ')]': - parens -= 1 - elif in_def and text == ':' and parens == 1: + paren_stack.append(text) + elif text in ')]' and paren_stack: + paren_stack.pop() + # def f(arg: tp = default): ... + elif text == ':' and in_def and paren_stack == ['(']: annotated_func_arg = True - elif parens == 1 and text == ',': + elif len(paren_stack) == 1 and text == ',': annotated_func_arg = False - elif parens and text == '=': - if annotated_func_arg and parens == 1: + elif paren_stack and text == '=': + if ( + # PEP 696 defaults always use spaced-style `=` + # type A[T = default] = ... + # def f[T = default](): ... + # class C[T = default](): ... + (in_generic and paren_stack == ['[']) or + (annotated_func_arg and paren_stack == ['(']) + ): require_space = True if start == prev_end: yield (prev_end, missing_message) @@ -1051,7 +1083,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): no_space = True if start != prev_end: yield (prev_end, message) - if not parens: + if not paren_stack: annotated_func_arg = False prev_end = end @@ -1122,6 +1154,22 @@ def imports_on_separate_lines(logical_line): yield found, "E401 multiple imports on one line" +_STRING_PREFIXES = frozenset(('u', 'U', 'b', 'B', 'r', 'R')) + + +def _is_string_literal(line): + if line: + first_char = line[0] + if first_char in _STRING_PREFIXES: + first_char = line[1] + return first_char == '"' or first_char == "'" + return False + + +_ALLOWED_KEYWORDS_IN_IMPORTS = ( + 'try', 'except', 'else', 'finally', 'with', 'if', 'elif') + + @register_check def module_imports_on_top_of_file( logical_line, indent_level, checker_state, noqa): @@ -1140,15 +1188,6 @@ def module_imports_on_top_of_file( Okay: if x:\n import os """ # noqa - def is_string_literal(line): - if line[0] in 'uUbB': - line = line[1:] - if line and line[0] in 'rR': - line = line[1:] - return line and (line[0] == '"' or line[0] == "'") - - allowed_keywords = ( - 'try', 'except', 'else', 'finally', 'with', 'if', 'elif') if indent_level: # Allow imports in conditional statement/function return @@ -1156,25 +1195,25 @@ def module_imports_on_top_of_file( return if noqa: return - line = logical_line - if line.startswith('import ') or line.startswith('from '): + if logical_line.startswith(('import ', 'from ')): if checker_state.get('seen_non_imports', False): yield 0, "E402 module level import not at top of file" - elif re.match(DUNDER_REGEX, line): - return - elif any(line.startswith(kw) for kw in allowed_keywords): - # Allow certain keywords intermixed with imports in order to - # support conditional or filtered importing - return - elif is_string_literal(line): - # The first literal is a docstring, allow it. Otherwise, report - # error. - if checker_state.get('seen_docstring', False): - checker_state['seen_non_imports'] = True + elif not checker_state.get('seen_non_imports', False): + if DUNDER_REGEX.match(logical_line): + return + elif logical_line.startswith(_ALLOWED_KEYWORDS_IN_IMPORTS): + # Allow certain keywords intermixed with imports in order to + # support conditional or filtered importing + return + elif _is_string_literal(logical_line): + # The first literal is a docstring, allow it. Otherwise, + # report error. + if checker_state.get('seen_docstring', False): + checker_state['seen_non_imports'] = True + else: + checker_state['seen_docstring'] = True else: - checker_state['seen_docstring'] = True - else: - checker_state['seen_non_imports'] = True + checker_state['seen_non_imports'] = True @register_check @@ -1268,6 +1307,8 @@ def explicit_line_join(logical_line, tokens): comment = True if start[0] != prev_start and parens and backslash and not comment: yield backslash, "E502 the backslash is redundant between brackets" + if start[0] != prev_start: + comment = False # Reset comment flag on newline if end[0] != prev_end: if line.rstrip('\r\n').endswith('\\'): backslash = (end[0], len(line.splitlines()[-1]) - 1) @@ -1587,6 +1628,29 @@ def ambiguous_identifier(logical_line, tokens): prev_start = start +# https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals +_PYTHON_3000_VALID_ESC = frozenset([ + '\n', + '\\', + '\'', + '"', + 'a', + 'b', + 'f', + 'n', + 'r', + 't', + 'v', + '0', '1', '2', '3', '4', '5', '6', '7', + 'x', + + # Escape sequences only recognized in string literals + 'N', + 'u', + 'U', +]) + + @register_check def python_3000_invalid_escape_sequence(logical_line, tokens, noqa): r"""Invalid escape sequences are deprecated in Python 3.6. @@ -1597,41 +1661,27 @@ def python_3000_invalid_escape_sequence(logical_line, tokens, noqa): if noqa: return - # https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals - valid = [ - '\n', - '\\', - '\'', - '"', - 'a', - 'b', - 'f', - 'n', - 'r', - 't', - 'v', - '0', '1', '2', '3', '4', '5', '6', '7', - 'x', - - # Escape sequences only recognized in string literals - 'N', - 'u', - 'U', - ] - prefixes = [] for token_type, text, start, _, _ in tokens: - if token_type in {tokenize.STRING, FSTRING_START}: + if ( + token_type == tokenize.STRING or + token_type == FSTRING_START or + token_type == TSTRING_START + ): # Extract string modifiers (e.g. u or r) prefixes.append(text[:text.index(text[-1])].lower()) - if token_type in {tokenize.STRING, FSTRING_MIDDLE}: + if ( + token_type == tokenize.STRING or + token_type == FSTRING_MIDDLE or + token_type == TSTRING_MIDDLE + ): if 'r' not in prefixes[-1]: start_line, start_col = start pos = text.find('\\') while pos >= 0: pos += 1 - if text[pos] not in valid: + if text[pos] not in _PYTHON_3000_VALID_ESC: line = start_line + text.count('\n', 0, pos) if line == start_line: col = start_col + pos @@ -1643,7 +1693,11 @@ def python_3000_invalid_escape_sequence(logical_line, tokens, noqa): ) pos = text.find('\\', pos + 1) - if token_type in {tokenize.STRING, FSTRING_END}: + if ( + token_type == tokenize.STRING or + token_type == FSTRING_END or + token_type == TSTRING_END + ): prefixes.pop() @@ -1714,7 +1768,7 @@ def readlines(filename): def stdin_get_value(): """Read the value from stdin.""" - return io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors='ignore').read() + return io.TextIOWrapper(sys.stdin.buffer, errors='ignore').read() noqa = lru_cache(512)(re.compile(r'# no(?:qa|pep8)\b', re.I).search) @@ -1841,7 +1895,7 @@ class Checker: self.max_line_length = options.max_line_length self.max_doc_length = options.max_doc_length self.indent_size = options.indent_size - self.fstring_start = 0 + self.fstring_start = self.tstring_start = 0 self.multiline = False # in a multiline string? self.hang_closing = options.hang_closing self.indent_size = options.indent_size @@ -1900,9 +1954,7 @@ class Checker: def run_check(self, check, argument_names): """Run a check plugin.""" - arguments = [] - for name in argument_names: - arguments.append(getattr(self, name)) + arguments = [getattr(self, name) for name in argument_names] return check(*arguments) def init_checker_state(self, name, argument_names): @@ -1938,8 +1990,11 @@ class Checker: continue if token_type == tokenize.STRING: text = mute_string(text) - elif token_type == FSTRING_MIDDLE: # pragma: >=3.12 cover - text = 'x' * len(text) + elif token_type in {FSTRING_MIDDLE, TSTRING_MIDDLE}: # pragma: >=3.12 cover # noqa: E501 + # fstring tokens are "unescaped" braces -- re-escape! + brace_count = text.count('{') + text.count('}') + text = 'x' * (len(text) + brace_count) + end = (end[0], end[1] + brace_count) if prev_row: (start_row, start_col) = start if prev_row != start_row: # different row @@ -2027,6 +2082,8 @@ class Checker: if token.type == FSTRING_START: # pragma: >=3.12 cover self.fstring_start = token.start[0] + elif token.type == TSTRING_START: # pragma: >=3.14 cover + self.tstring_start = token.start[0] # a newline token ends a single physical line. elif _is_eol_token(token): # if the file does not end with a newline, the NEWLINE @@ -2038,7 +2095,8 @@ class Checker: self.check_physical(token.line) elif ( token.type == tokenize.STRING and '\n' in token.string or - token.type == FSTRING_END + token.type == FSTRING_END or + token.type == TSTRING_END ): # Less obviously, a string that contains newlines is a # multiline string, either triple-quoted or with internal @@ -2059,6 +2117,8 @@ class Checker: return if token.type == FSTRING_END: # pragma: >=3.12 cover start = self.fstring_start + elif token.type == TSTRING_END: # pragma: >=3.12 cover + start = self.tstring_start else: start = token.start[0] end = token.end[0]