From d616d70c504ed3f63aece45f42f9c53d80e92ca9 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Sun, 6 Sep 2015 20:04:50 +0300 Subject: [PATCH 01/15] Parse and highlight the 'async' keyword (PY-16094) We had to switch to stack-based parsing contexts in order to be able to use context-sensitive token filtering for 'async'. Also this commit fixes the highlighting annotator issue with blinking names of the functions and blinking 'async'. --- .../plugins/ipnb/psi/IpnbPyParser.java | 2 +- .../com/jetbrains/python/PyTokenTypes.java | 1 + python/src/META-INF/python-core.xml | 2 + .../python/PythonTokenSetContributor.java | 2 +- .../console/parsing/PyConsoleParser.java | 2 +- .../parsing/PyConsoleParsingContext.java | 9 +- .../python/parsing/FunctionParsing.java | 23 +- .../python/parsing/ParsingContext.java | 21 ++ .../python/parsing/ParsingScope.java | 14 ++ .../jetbrains/python/parsing/PyParser.java | 9 +- .../python/parsing/StatementParsing.java | 211 ++++++++++-------- .../DumbAwareHighlightingAnnotator.java | 62 +++++ python/testData/highlighting/async.py | 28 +++ .../highlighting/returnOutsideOfFunction.py | 4 +- .../returnWithArgumentsInGenerator.py | 2 +- python/testData/psi/AsyncDef.py | 15 ++ python/testData/psi/AsyncDef.txt | 80 +++++++ python/testData/psi/AsyncFor.py | 7 + python/testData/psi/AsyncFor.txt | 54 +++++ python/testData/psi/AsyncWith.py | 6 + python/testData/psi/AsyncWith.txt | 44 ++++ .../python/PythonHighlightingTest.java | 4 + .../jetbrains/python/PythonParsingTest.java | 12 + 23 files changed, 503 insertions(+), 111 deletions(-) create mode 100644 python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java create mode 100644 python/testData/highlighting/async.py create mode 100644 python/testData/psi/AsyncDef.py create mode 100644 python/testData/psi/AsyncDef.txt create mode 100644 python/testData/psi/AsyncFor.py create mode 100644 python/testData/psi/AsyncFor.txt create mode 100644 python/testData/psi/AsyncWith.py create mode 100644 python/testData/psi/AsyncWith.txt diff --git a/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java b/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java index 53e0de51af83..093b03a3b972 100644 --- a/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java +++ b/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java @@ -32,7 +32,7 @@ public class IpnbPyParser extends PyParser { builder.setTokenTypeRemapper(statementParser); while (!builder.eof()) { - statementParser.parseStatement(context.emptyParsingScope()); + statementParser.parseStatement(); } rootMarker.done(root); return builder.getTreeBuilt(); diff --git a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java index 5d3060e155a6..be858df92181 100644 --- a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java +++ b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java @@ -73,6 +73,7 @@ public class PyTokenTypes { public static final PyElementType FALSE_KEYWORD = new PyElementType("FALSE_KEYWORD"); public static final PyElementType NONLOCAL_KEYWORD = new PyElementType("NONLOCAL_KEYWORD"); public static final PyElementType DEBUG_KEYWORD = new PyElementType("DEBUG_KEYWORD"); + public static final PyElementType ASYNC_KEYWORD = new PyElementType("ASYNC_KEYWORD"); public static final PyElementType INTEGER_LITERAL = new PyElementType("INTEGER_LITERAL"); public static final PyElementType FLOAT_LITERAL = new PyElementType("FLOAT_LITERAL"); diff --git a/python/src/META-INF/python-core.xml b/python/src/META-INF/python-core.xml index 29489c86a618..3ed392a62d48 100644 --- a/python/src/META-INF/python-core.xml +++ b/python/src/META-INF/python-core.xml @@ -41,6 +41,7 @@ + @@ -614,6 +615,7 @@ + diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 891eff28c507..51a42589109b 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -73,7 +73,7 @@ public class PythonTokenSetContributor extends PythonDialectsTokenSetContributor LAMBDA_KEYWORD, NOT_KEYWORD, OR_KEYWORD, PASS_KEYWORD, PRINT_KEYWORD, RAISE_KEYWORD, RETURN_KEYWORD, TRY_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, YIELD_KEYWORD, - NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD); + NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD, ASYNC_KEYWORD); } @NotNull diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java index 100330608c73..2d71ee03a685 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java @@ -56,7 +56,7 @@ public class PyConsoleParser extends PyParser{ builder.setTokenTypeRemapper(stmt_parser); // must be done before touching the caching lexer with eof() call. while (!builder.eof()) { - stmt_parser.parseStatement(context.emptyParsingScope()); + stmt_parser.parseStatement(); } rootMarker.done(root); return builder.getTreeBuilt(); diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java index 32e8882b38b7..8c6aa7e33b8c 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java @@ -72,7 +72,7 @@ public class PyConsoleParsingContext extends ParsingContext { @Override - public void parseStatement(ParsingScope scope) { + public void parseStatement() { if (myStartsWithIPythonSymbol) { parseIPythonCommand(); } @@ -89,7 +89,7 @@ public class PyConsoleParsingContext extends ParsingContext { myBuilder.advanceLexer(); } } - super.parseStatement(scope); + super.parseStatement(); } } @@ -101,13 +101,14 @@ public class PyConsoleParsingContext extends ParsingContext { ipythonCommand.done(PyElementTypes.EMPTY_EXPRESSION); } - protected void checkEndOfStatement(ParsingScope scope) { + protected void checkEndOfStatement() { if (myPythonConsoleData.isIPythonEnabled()) { PsiBuilder builder = myContext.getBuilder(); if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { builder.advanceLexer(); } else if (builder.getTokenType() == PyTokenTypes.SEMICOLON) { + final ParsingScope scope = getParsingContext().getScope(); if (!scope.isSuite()) { builder.advanceLexer(); if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { @@ -131,7 +132,7 @@ public class PyConsoleParsingContext extends ParsingContext { } } else { - super.checkEndOfStatement(scope); + super.checkEndOfStatement(); } } } diff --git a/python/src/com/jetbrains/python/parsing/FunctionParsing.java b/python/src/com/jetbrains/python/parsing/FunctionParsing.java index 60b32e91292c..2c8b7dff584b 100644 --- a/python/src/com/jetbrains/python/parsing/FunctionParsing.java +++ b/python/src/com/jetbrains/python/parsing/FunctionParsing.java @@ -20,6 +20,7 @@ import com.intellij.lang.WhitespacesBinders; import com.intellij.psi.tree.IElementType; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import org.jetbrains.annotations.NotNull; import static com.jetbrains.python.PyBundle.message; @@ -33,23 +34,25 @@ public class FunctionParsing extends Parsing { super(context); } - public void parseFunctionDeclaration() { + public void parseFunctionDeclaration(@NotNull PsiBuilder.Marker endMarker) { assertCurrentToken(PyTokenTypes.DEF_KEYWORD); - final PsiBuilder.Marker functionMarker = myBuilder.mark(); - parseFunctionInnards(functionMarker); + parseFunctionInnards(endMarker); } protected IElementType getFunctionType() { return FUNCTION_TYPE; } - protected void parseFunctionInnards(PsiBuilder.Marker functionMarker) { + protected void parseFunctionInnards(@NotNull PsiBuilder.Marker functionMarker) { myBuilder.advanceLexer(); parseIdentifierOrSkip(PyTokenTypes.LPAR); parseParameterList(); parseReturnTypeAnnotation(); checkMatches(PyTokenTypes.COLON, message("PARSE.expected.colon")); - getStatementParser().parseSuite(functionMarker, getFunctionType(), myContext.emptyParsingScope().withFunction(true)); + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withFunction(true)); + getStatementParser().parseSuite(functionMarker, getFunctionType()); + context.popScope(); } public void parseReturnTypeAnnotation() { @@ -63,7 +66,7 @@ public class FunctionParsing extends Parsing { } } - public void parseDecoratedDeclaration(ParsingScope scope) { + public void parseDecoratedDeclaration() { assertCurrentToken(PyTokenTypes.AT); // ??? need this? final PsiBuilder.Marker decoratorStartMarker = myBuilder.mark(); final PsiBuilder.Marker decoListMarker = myBuilder.mark(); @@ -92,15 +95,15 @@ public class FunctionParsing extends Parsing { } if (decorated) decoListMarker.done(PyElementTypes.DECORATOR_LIST); //else decoListMarker.rollbackTo(); - parseDeclarationAfterDecorator(decoratorStartMarker, scope); + parseDeclarationAfterDecorator(decoratorStartMarker); } - protected void parseDeclarationAfterDecorator(PsiBuilder.Marker endMarker, ParsingScope scope) { + protected void parseDeclarationAfterDecorator(PsiBuilder.Marker endMarker) { if (myBuilder.getTokenType() == PyTokenTypes.DEF_KEYWORD) { - parseFunctionInnards(endMarker); // it calls endMarker.done() + parseFunctionInnards(endMarker); } else if (myBuilder.getTokenType() == PyTokenTypes.CLASS_KEYWORD) { - getStatementParser().parseClassDeclaration(endMarker, scope); + getStatementParser().parseClassDeclaration(endMarker); } else { myBuilder.error(message("PARSE.expected.@.or.def")); diff --git a/python/src/com/jetbrains/python/parsing/ParsingContext.java b/python/src/com/jetbrains/python/parsing/ParsingContext.java index 66f3ebb32e62..94253c07720a 100644 --- a/python/src/com/jetbrains/python/parsing/ParsingContext.java +++ b/python/src/com/jetbrains/python/parsing/ParsingContext.java @@ -17,6 +17,10 @@ package com.jetbrains.python.parsing; import com.intellij.lang.PsiBuilder; import com.jetbrains.python.psi.LanguageLevel; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayDeque; +import java.util.Deque; public class ParsingContext { private final StatementParsing stmtParser; @@ -24,6 +28,7 @@ public class ParsingContext { private final FunctionParsing functionParser; private final PsiBuilder myBuilder; private final LanguageLevel myLanguageLevel; + private final Deque myScopes; public ParsingContext(final PsiBuilder builder, LanguageLevel languageLevel, StatementParsing.FUTURE futureFlag) { myBuilder = builder; @@ -31,6 +36,22 @@ public class ParsingContext { stmtParser = new StatementParsing(this, futureFlag); expressionParser = new ExpressionParsing(this); functionParser = new FunctionParsing(this); + myScopes = new ArrayDeque(); + myScopes.push(emptyParsingScope()); + } + + @NotNull + public ParsingScope popScope() { + return myScopes.pop(); + } + + public void pushScope(@NotNull ParsingScope scope) { + myScopes.push(scope); + } + + @NotNull + public ParsingScope getScope() { + return myScopes.peek(); } public StatementParsing getStatementParser() { diff --git a/python/src/com/jetbrains/python/parsing/ParsingScope.java b/python/src/com/jetbrains/python/parsing/ParsingScope.java index f029c2863a9a..d73721d52751 100644 --- a/python/src/com/jetbrains/python/parsing/ParsingScope.java +++ b/python/src/com/jetbrains/python/parsing/ParsingScope.java @@ -23,6 +23,9 @@ public class ParsingScope { private boolean myClass = false; private boolean mySuite = false; private boolean myAfterSemicolon = false; + private boolean myAsync = false; + + protected ParsingScope() {} public ParsingScope withFunction(boolean flag) { final ParsingScope result = copy(); @@ -42,6 +45,12 @@ public class ParsingScope { return result; } + public ParsingScope withAsync() { + final ParsingScope result = copy(); + result.myAsync = true; + return result; + } + public boolean isFunction() { return myFunction; } @@ -54,6 +63,10 @@ public class ParsingScope { return mySuite; } + public boolean isAsync() { + return myAsync; + } + public boolean isAfterSemicolon() { return myAfterSemicolon; } @@ -71,6 +84,7 @@ public class ParsingScope { result.myFunction = myFunction; result.myClass = myClass; result.mySuite = mySuite; + result.myAsync = myAsync; return result; } } diff --git a/python/src/com/jetbrains/python/parsing/PyParser.java b/python/src/com/jetbrains/python/parsing/PyParser.java index d7e431af46e3..631485670de8 100644 --- a/python/src/com/jetbrains/python/parsing/PyParser.java +++ b/python/src/com/jetbrains/python/parsing/PyParser.java @@ -49,14 +49,15 @@ public class PyParser implements PsiParser { builder.setTokenTypeRemapper(statementParser); // must be done before touching the caching lexer with eof() call. boolean lastAfterSemicolon = false; while (!builder.eof()) { - ParsingScope scope = context.emptyParsingScope(); + context.pushScope(context.emptyParsingScope()); if (lastAfterSemicolon) { - statementParser.parseSimpleStatement(scope); + statementParser.parseSimpleStatement(); } else { - statementParser.parseStatement(scope); + statementParser.parseStatement(); } - lastAfterSemicolon = scope.isAfterSemicolon(); + lastAfterSemicolon = context.getScope().isAfterSemicolon(); + context.popScope(); } rootMarker.done(root); ASTNode ast = builder.getTreeBuilt(); diff --git a/python/src/com/jetbrains/python/parsing/StatementParsing.java b/python/src/com/jetbrains/python/parsing/StatementParsing.java index 8de31cd09f27..bd6b423c033e 100644 --- a/python/src/com/jetbrains/python/parsing/StatementParsing.java +++ b/python/src/com/jetbrains/python/parsing/StatementParsing.java @@ -23,6 +23,7 @@ import com.intellij.psi.tree.TokenSet; import com.intellij.util.text.CharArrayUtil; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.psi.PyElementType; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.Nullable; @@ -49,6 +50,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { @NonNls protected static final String TOK_FALSE = "False"; @NonNls protected static final String TOK_NONLOCAL = "nonlocal"; @NonNls protected static final String TOK_EXEC = "exec"; + @NonNls protected static final String TOK_ASYNC = "async"; private static final String EXPRESSION_EXPECTED = "Expression expected"; public static final String IDENTIFIER_EXPECTED = "Identifier expected"; @@ -86,7 +88,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myBuilder.setTokenTypeRemapper(this); // clear cached token type } - public void parseStatement(ParsingScope scope) { + public void parseStatement() { while (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { myBuilder.advanceLexer(); @@ -98,103 +100,107 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (firstToken == null) return; if (firstToken == PyTokenTypes.WHILE_KEYWORD) { - parseWhileStatement(scope); + parseWhileStatement(); return; } if (firstToken == PyTokenTypes.IF_KEYWORD) { - parseIfStatement(PyTokenTypes.IF_KEYWORD, PyTokenTypes.ELIF_KEYWORD, PyTokenTypes.ELSE_KEYWORD, PyElementTypes.IF_STATEMENT, scope); + parseIfStatement(PyTokenTypes.IF_KEYWORD, PyTokenTypes.ELIF_KEYWORD, PyTokenTypes.ELSE_KEYWORD, PyElementTypes.IF_STATEMENT); return; } if (firstToken == PyTokenTypes.FOR_KEYWORD) { - parseForStatement(scope); + parseForStatement(myBuilder.mark()); return; } if (firstToken == PyTokenTypes.TRY_KEYWORD) { - parseTryStatement(scope); + parseTryStatement(); return; } if (firstToken == PyTokenTypes.DEF_KEYWORD) { - getFunctionParser().parseFunctionDeclaration(); + getFunctionParser().parseFunctionDeclaration(myBuilder.mark()); return; } if (firstToken == PyTokenTypes.AT) { - getFunctionParser().parseDecoratedDeclaration(scope); + getFunctionParser().parseDecoratedDeclaration(); return; } if (firstToken == PyTokenTypes.CLASS_KEYWORD) { - parseClassDeclaration(scope); + parseClassDeclaration(); return; } if (firstToken == PyTokenTypes.WITH_KEYWORD) { - parseWithStatement(scope); + parseWithStatement(myBuilder.mark()); + return; + } + if (firstToken == PyTokenTypes.ASYNC_KEYWORD) { + parseAsyncStatement(); return; } - parseSimpleStatement(scope); + parseSimpleStatement(); } - protected void parseSimpleStatement(ParsingScope scope) { + protected void parseSimpleStatement() { PsiBuilder builder = myContext.getBuilder(); final IElementType firstToken = builder.getTokenType(); if (firstToken == null) { return; } if (firstToken == PyTokenTypes.PRINT_KEYWORD && hasPrintStatement()) { - parsePrintStatement(builder, scope); + parsePrintStatement(builder); return; } if (firstToken == PyTokenTypes.ASSERT_KEYWORD) { - parseAssertStatement(scope); + parseAssertStatement(); return; } if (firstToken == PyTokenTypes.BREAK_KEYWORD) { - parseKeywordStatement(builder, PyElementTypes.BREAK_STATEMENT, scope); + parseKeywordStatement(builder, PyElementTypes.BREAK_STATEMENT); return; } if (firstToken == PyTokenTypes.CONTINUE_KEYWORD) { - parseKeywordStatement(builder, PyElementTypes.CONTINUE_STATEMENT, scope); + parseKeywordStatement(builder, PyElementTypes.CONTINUE_STATEMENT); return; } if (firstToken == PyTokenTypes.DEL_KEYWORD) { - parseDelStatement(scope); + parseDelStatement(); return; } if (firstToken == PyTokenTypes.EXEC_KEYWORD) { - parseExecStatement(scope); + parseExecStatement(); return; } if (firstToken == PyTokenTypes.GLOBAL_KEYWORD) { - parseNameDefiningStatement(scope, PyElementTypes.GLOBAL_STATEMENT); + parseNameDefiningStatement(PyElementTypes.GLOBAL_STATEMENT); return; } if (firstToken == PyTokenTypes.NONLOCAL_KEYWORD) { - parseNameDefiningStatement(scope, PyElementTypes.NONLOCAL_STATEMENT); + parseNameDefiningStatement(PyElementTypes.NONLOCAL_STATEMENT); return; } if (firstToken == PyTokenTypes.IMPORT_KEYWORD) { - parseImportStatement(scope, PyElementTypes.IMPORT_STATEMENT, PyElementTypes.IMPORT_ELEMENT); + parseImportStatement(PyElementTypes.IMPORT_STATEMENT, PyElementTypes.IMPORT_ELEMENT); return; } if (firstToken == PyTokenTypes.FROM_KEYWORD) { - parseFromImportStatement(scope); + parseFromImportStatement(); return; } if (firstToken == PyTokenTypes.PASS_KEYWORD) { - parseKeywordStatement(builder, PyElementTypes.PASS_STATEMENT, scope); + parseKeywordStatement(builder, PyElementTypes.PASS_STATEMENT); return; } if (firstToken == PyTokenTypes.RETURN_KEYWORD) { - parseReturnStatement(builder, scope); + parseReturnStatement(builder); return; } if (firstToken == PyTokenTypes.RAISE_KEYWORD) { - parseRaiseStatement(scope); + parseRaiseStatement(); return; } PsiBuilder.Marker exprStatement = builder.mark(); if (builder.getTokenType() == PyTokenTypes.YIELD_KEYWORD) { getExpressionParser().parseYieldOrTupleExpression(false); - checkEndOfStatement(scope); + checkEndOfStatement(); exprStatement.done(PyElementTypes.EXPRESSION_STATEMENT); return; } @@ -242,7 +248,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - checkEndOfStatement(scope); + checkEndOfStatement(); exprStatement.done(statementType); return; } @@ -270,8 +276,9 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { return myContext.getLanguageLevel().hasPrintStatement() && !myFutureFlags.contains(FUTURE.PRINT_FUNCTION); } - protected void checkEndOfStatement(ParsingScope scope) { + protected void checkEndOfStatement() { PsiBuilder builder = myContext.getBuilder(); + final ParsingScope scope = getParsingContext().getScope(); if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { builder.advanceLexer(); scope.setAfterSemicolon(false); @@ -291,7 +298,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - private void parsePrintStatement(final PsiBuilder builder, ParsingScope scope) { + private void parsePrintStatement(final PsiBuilder builder) { LOG.assertTrue(builder.getTokenType() == PyTokenTypes.PRINT_KEYWORD); final PsiBuilder.Marker statement = builder.mark(); builder.advanceLexer(); @@ -311,29 +318,29 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } getExpressionParser().parseSingleExpression(false); } - checkEndOfStatement(scope); + checkEndOfStatement(); statement.done(PyElementTypes.PRINT_STATEMENT); } - protected void parseKeywordStatement(PsiBuilder builder, IElementType statementType, ParsingScope scope) { + protected void parseKeywordStatement(PsiBuilder builder, IElementType statementType) { final PsiBuilder.Marker statement = builder.mark(); builder.advanceLexer(); - checkEndOfStatement(scope); + checkEndOfStatement(); statement.done(statementType); } - private void parseReturnStatement(PsiBuilder builder, ParsingScope inSuite) { + private void parseReturnStatement(PsiBuilder builder) { LOG.assertTrue(builder.getTokenType() == PyTokenTypes.RETURN_KEYWORD); final PsiBuilder.Marker returnStatement = builder.mark(); builder.advanceLexer(); if (builder.getTokenType() != null && !getEndOfStatementsTokens().contains(builder.getTokenType())) { getExpressionParser().parseExpression(); } - checkEndOfStatement(inSuite); + checkEndOfStatement(); returnStatement.done(PyElementTypes.RETURN_STATEMENT); } - private void parseDelStatement(ParsingScope inSuite) { + private void parseDelStatement() { assertCurrentToken(PyTokenTypes.DEL_KEYWORD); final PsiBuilder.Marker delStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -349,11 +356,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - checkEndOfStatement(inSuite); + checkEndOfStatement(); delStatement.done(PyElementTypes.DEL_STATEMENT); } - private void parseRaiseStatement(ParsingScope inSuite) { + private void parseRaiseStatement() { assertCurrentToken(PyTokenTypes.RAISE_KEYWORD); final PsiBuilder.Marker raiseStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -374,11 +381,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } } - checkEndOfStatement(inSuite); + checkEndOfStatement(); raiseStatement.done(PyElementTypes.RAISE_STATEMENT); } - private void parseAssertStatement(ParsingScope scope) { + private void parseAssertStatement() { assertCurrentToken(PyTokenTypes.ASSERT_KEYWORD); final PsiBuilder.Marker assertStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -389,7 +396,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myContext.getBuilder().error(EXPRESSION_EXPECTED); } } - checkEndOfStatement(scope); + checkEndOfStatement(); } else { myContext.getBuilder().error(EXPRESSION_EXPECTED); @@ -397,12 +404,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { assertStatement.done(PyElementTypes.ASSERT_STATEMENT); } - protected void parseImportStatement(ParsingScope scope, IElementType statementType, IElementType elementType) { + protected void parseImportStatement(IElementType statementType, IElementType elementType) { final PsiBuilder builder = myContext.getBuilder(); final PsiBuilder.Marker importStatement = builder.mark(); builder.advanceLexer(); parseImportElements(elementType, true, false, false); - checkEndOfStatement(scope); + checkEndOfStatement(); importStatement.done(statementType); } @@ -411,7 +418,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { from identifier import id, id... -- may be either relative or absolute from . import identifier -- only relative */ - private void parseFromImportStatement(ParsingScope inSuite) { + private void parseFromImportStatement() { PsiBuilder builder = myContext.getBuilder(); assertCurrentToken(PyTokenTypes.FROM_KEYWORD); myFutureImportPhase = Phase.FROM; @@ -447,7 +454,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { statementType = types.statement; parseImportElements(types.element, false, false, from_future); } - checkEndOfStatement(inSuite); + checkEndOfStatement(); fromImportStatement.done(statementType); myFutureImportPhase = Phase.NONE; } @@ -563,7 +570,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { return true; } - private void parseNameDefiningStatement(ParsingScope scope, final PyElementType elementType) { + private void parseNameDefiningStatement(final PyElementType elementType) { final PsiBuilder.Marker globalStatement = myBuilder.mark(); myBuilder.advanceLexer(); parseIdentifier(PyElementTypes.TARGET_EXPRESSION); @@ -571,11 +578,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myBuilder.advanceLexer(); parseIdentifier(PyElementTypes.TARGET_EXPRESSION); } - checkEndOfStatement(scope); + checkEndOfStatement(); globalStatement.done(elementType); } - private void parseExecStatement(ParsingScope inSuite) { + private void parseExecStatement() { assertCurrentToken(PyTokenTypes.EXEC_KEYWORD); final PsiBuilder.Marker execStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -588,12 +595,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { getExpressionParser().parseSingleExpression(false); } } - checkEndOfStatement(inSuite); + checkEndOfStatement(); execStatement.done(PyElementTypes.EXEC_STATEMENT); } - protected void parseIfStatement(PyElementType ifKeyword, PyElementType elifKeyword, PyElementType elseKeyword, PyElementType elementType, - ParsingScope scope) { + protected void parseIfStatement(PyElementType ifKeyword, PyElementType elifKeyword, PyElementType elseKeyword, + PyElementType elementType) { assertCurrentToken(ifKeyword); final PsiBuilder.Marker ifStatement = myBuilder.mark(); final PsiBuilder.Marker ifPart = myBuilder.mark(); @@ -601,7 +608,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (!getExpressionParser().parseSingleExpression(false)) { myBuilder.error("expression expected"); } - parseColonAndSuite(scope); + parseColonAndSuite(); ifPart.done(PyElementTypes.IF_PART_IF); PsiBuilder.Marker elifPart = myBuilder.mark(); while (myBuilder.getTokenType() == elifKeyword) { @@ -609,7 +616,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (!getExpressionParser().parseSingleExpression(false)) { myBuilder.error("expression expected"); } - parseColonAndSuite(scope); + parseColonAndSuite(); elifPart.done(PyElementTypes.IF_PART_ELIF); elifPart = myBuilder.mark(); } @@ -617,7 +624,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == elseKeyword) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { @@ -647,33 +654,32 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { return result; } - private void parseForStatement(ParsingScope scope) { + private void parseForStatement(PsiBuilder.Marker endMarker) { assertCurrentToken(PyTokenTypes.FOR_KEYWORD); - final PsiBuilder.Marker statement = myBuilder.mark(); - parseForPart(scope); + parseForPart(); final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { elsePart.drop(); } - statement.done(PyElementTypes.FOR_STATEMENT); + endMarker.done(PyElementTypes.FOR_STATEMENT); } - protected void parseForPart(ParsingScope scope) { + protected void parseForPart() { final PsiBuilder.Marker forPart = myBuilder.mark(); myBuilder.advanceLexer(); getExpressionParser().parseExpression(true, true); checkMatches(PyTokenTypes.IN_KEYWORD, "'in' expected"); getExpressionParser().parseExpression(); - parseColonAndSuite(scope); + parseColonAndSuite(); forPart.done(PyElementTypes.FOR_PART); } - private void parseWhileStatement(ParsingScope scope) { + private void parseWhileStatement() { assertCurrentToken(PyTokenTypes.WHILE_KEYWORD); final PsiBuilder.Marker statement = myBuilder.mark(); final PsiBuilder.Marker whilePart = myBuilder.mark(); @@ -681,12 +687,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (!getExpressionParser().parseSingleExpression(false)) { myBuilder.error(EXPRESSION_EXPECTED); } - parseColonAndSuite(scope); + parseColonAndSuite(); whilePart.done(PyElementTypes.WHILE_PART); final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { @@ -695,12 +701,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { statement.done(PyElementTypes.WHILE_STATEMENT); } - private void parseTryStatement(ParsingScope scope) { + private void parseTryStatement() { assertCurrentToken(PyTokenTypes.TRY_KEYWORD); final PsiBuilder.Marker statement = myBuilder.mark(); final PsiBuilder.Marker tryPart = myBuilder.mark(); myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); tryPart.done(PyElementTypes.TRY_PART); boolean haveExceptClause = false; if (myBuilder.getTokenType() == PyTokenTypes.EXCEPT_KEYWORD) { @@ -720,13 +726,13 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } } - parseColonAndSuite(scope); + parseColonAndSuite(); exceptBlock.done(PyElementTypes.EXCEPT_PART); } final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { @@ -736,7 +742,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker finallyPart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.FINALLY_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); finallyPart.done(PyElementTypes.FINALLY_PART); } else { @@ -750,9 +756,9 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { statement.done(PyElementTypes.TRY_EXCEPT_STATEMENT); } - private void parseColonAndSuite(ParsingScope scope) { + private void parseColonAndSuite() { if (expectColon()) { - parseSuite(scope); + parseSuite(); } else { final PsiBuilder.Marker mark = myBuilder.mark(); @@ -760,9 +766,8 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - private void parseWithStatement(ParsingScope scope) { + private void parseWithStatement(PsiBuilder.Marker endMarker) { assertCurrentToken(PyTokenTypes.WITH_KEYWORD); - final PsiBuilder.Marker statement = myBuilder.mark(); myBuilder.advanceLexer(); while (true) { PsiBuilder.Marker withItem = myBuilder.mark(); @@ -780,16 +785,16 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { break; } } - parseColonAndSuite(scope); - statement.done(PyElementTypes.WITH_STATEMENT); + parseColonAndSuite(); + endMarker.done(PyElementTypes.WITH_STATEMENT); } - private void parseClassDeclaration(ParsingScope scope) { + private void parseClassDeclaration() { final PsiBuilder.Marker classMarker = myBuilder.mark(); - parseClassDeclaration(classMarker, scope); + parseClassDeclaration(classMarker); } - public void parseClassDeclaration(PsiBuilder.Marker classMarker, ParsingScope scope) { + public void parseClassDeclaration(PsiBuilder.Marker classMarker) { assertCurrentToken(PyTokenTypes.CLASS_KEYWORD); myBuilder.advanceLexer(); parseIdentifierOrSkip(PyTokenTypes.LPAR, PyTokenTypes.COLON); @@ -800,15 +805,41 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker inheritMarker = myBuilder.mark(); inheritMarker.done(PyElementTypes.ARGUMENT_LIST); } - parseColonAndSuite(scope.withClass(true)); + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withClass(true)); + parseColonAndSuite(); + context.popScope(); classMarker.done(PyElementTypes.CLASS_DECLARATION); } - public void parseSuite(ParsingScope scope) { - parseSuite(null, null, scope); + private void parseAsyncStatement() { + assertCurrentToken(PyTokenTypes.ASYNC_KEYWORD); + final PsiBuilder.Marker marker = myBuilder.mark(); + myBuilder.advanceLexer(); + final IElementType token = myBuilder.getTokenType(); + if (token == PyTokenTypes.DEF_KEYWORD) { + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withAsync()); + getFunctionParser().parseFunctionDeclaration(marker); + context.popScope(); + } + else if (token == PyTokenTypes.WITH_KEYWORD) { + parseWithStatement(marker); + } + else if (token == PyTokenTypes.FOR_KEYWORD) { + parseForStatement(marker); + } + else { + marker.drop(); + myBuilder.error("'def' or 'with' or 'for' expected"); + } } - public void parseSuite(@Nullable PsiBuilder.Marker endMarker, @Nullable IElementType elType, ParsingScope scope) { + public void parseSuite() { + parseSuite(null, null); + } + + public void parseSuite(@Nullable PsiBuilder.Marker endMarker, @Nullable IElementType elType) { if (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { myBuilder.advanceLexer(); @@ -821,7 +852,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } else { while (!myBuilder.eof() && myBuilder.getTokenType() != PyTokenTypes.DEDENT) { - parseStatement(scope); + parseStatement(); } } } @@ -837,10 +868,6 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (indentFound && !myBuilder.eof()) { checkMatches(PyTokenTypes.DEDENT, "Dedent expected"); } - // NOTE: the following line advances the PsiBuilder lexer and thus - // ensures that the whitespace following the statement list is included - // in the block containing the statement list - myBuilder.getTokenType(); } else { final PsiBuilder.Marker marker = myBuilder.mark(); @@ -848,12 +875,17 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myBuilder.error("Statement expected"); } else { - parseSimpleStatement(scope.withSuite(true)); + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withSuite(true)); + parseSimpleStatement(); + context.popScope(); while (matchToken(PyTokenTypes.SEMICOLON)) { if (matchToken(PyTokenTypes.STATEMENT_BREAK)) { break; } - parseSimpleStatement(scope.withSuite(true)); + context.pushScope(context.getScope().withSuite(true)); + parseSimpleStatement(); + context.popScope(); } } marker.done(PyElementTypes.STATEMENT_LIST); @@ -905,6 +937,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (isWordAtPosition(text, start, end, TOK_NONLOCAL)) { return PyTokenTypes.NONLOCAL_KEYWORD; } + if (myContext.getLanguageLevel().isAtLeast(LanguageLevel.PYTHON35) && isWordAtPosition(text, start, end, TOK_ASYNC)) { + if (myContext.getScope().isAsync() || myBuilder.lookAhead(1) == PyTokenTypes.DEF_KEYWORD) { + return PyTokenTypes.ASYNC_KEYWORD; + } + } } else if (!myContext.getLanguageLevel().isPy3K() && source == PyTokenTypes.IDENTIFIER) { if (isWordAtPosition(text, start, end, TOK_EXEC)) { diff --git a/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java new file mode 100644 index 000000000000..717d59f8e37d --- /dev/null +++ b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java @@ -0,0 +1,62 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.validation; + +import com.intellij.codeInsight.daemon.impl.HighlightRangeExtension; +import com.intellij.lang.ASTNode; +import com.intellij.lang.annotation.Annotation; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.highlighting.PyHighlighter; +import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.psi.PyForStatement; +import com.jetbrains.python.psi.PyFunction; +import com.jetbrains.python.psi.PyWithStatement; +import org.jetbrains.annotations.NotNull; + +/** + * @author vlan + */ +public class DumbAwareHighlightingAnnotator extends PyAnnotator implements HighlightRangeExtension { + @Override + public void visitPyFunction(PyFunction node) { + highlightAsyncKeyword(node); + } + + @Override + public void visitPyForStatement(PyForStatement node) { + highlightAsyncKeyword(node); + } + + @Override + public void visitPyWithStatement(PyWithStatement node) { + highlightAsyncKeyword(node); + } + + @Override + public boolean isForceHighlightParents(@NotNull PsiFile file) { + return file instanceof PyFile; + } + + private void highlightAsyncKeyword(@NotNull PsiElement node) { + final ASTNode asyncNode = node.getNode().findChildByType(PyTokenTypes.ASYNC_KEYWORD); + if (asyncNode != null) { + final Annotation annotation = getHolder().createInfoAnnotation(asyncNode, null); + annotation.setTextAttributes(PyHighlighter.PY_KEYWORD); + } + } +} diff --git a/python/testData/highlighting/async.py b/python/testData/highlighting/async.py new file mode 100644 index 000000000000..545f5ebf2c91 --- /dev/null +++ b/python/testData/highlighting/async.py @@ -0,0 +1,28 @@ +async def foo(): + pass + +async = 1 + +async def bar(): + pass + + +async def # Incomplete + + +def regular(xs): + + async def quux(): + async for x in xs: + pass + + async with xs: + pass + + async for x in xs: + pass + + async with xs: + pass + + return async diff --git a/python/testData/highlighting/returnOutsideOfFunction.py b/python/testData/highlighting/returnOutsideOfFunction.py index 8ce3bb17b73d..78054559568a 100644 --- a/python/testData/highlighting/returnOutsideOfFunction.py +++ b/python/testData/highlighting/returnOutsideOfFunction.py @@ -1,3 +1,3 @@ -def foo(): - class C: +def foo(): + class C: return 1 \ No newline at end of file diff --git a/python/testData/highlighting/returnWithArgumentsInGenerator.py b/python/testData/highlighting/returnWithArgumentsInGenerator.py index f16d817498ed..4c5ba91e5aba 100644 --- a/python/testData/highlighting/returnWithArgumentsInGenerator.py +++ b/python/testData/highlighting/returnWithArgumentsInGenerator.py @@ -1,3 +1,3 @@ -def f(): +def f(): yield 42 return 28 diff --git a/python/testData/psi/AsyncDef.py b/python/testData/psi/AsyncDef.py new file mode 100644 index 000000000000..4a7b81c00582 --- /dev/null +++ b/python/testData/psi/AsyncDef.py @@ -0,0 +1,15 @@ +async def foo(x, y): + pass + + async def foo_nested(): + pass + + +async = 10 + + +def bar(): + print(async) + + async def bar_nested(): + pass diff --git a/python/testData/psi/AsyncDef.txt b/python/testData/psi/AsyncDef.txt new file mode 100644 index 000000000000..15792ae90d92 --- /dev/null +++ b/python/testData/psi/AsyncDef.txt @@ -0,0 +1,80 @@ +PyFile:AsyncDef.py + PyFunction('foo') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PyNamedParameter('x') + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyNamedParameter('y') + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n ') + PyFunction('foo_nested') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo_nested') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n\n') + PyAssignmentStatement + PyTargetExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('10') + PsiWhiteSpace('\n\n\n') + PyFunction('bar') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyExpressionStatement + PyCallExpression: print + PyReferenceExpression: print + PsiElement(Py:IDENTIFIER)('print') + PyArgumentList + PsiElement(Py:LPAR)('(') + PyReferenceExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiElement(Py:RPAR)(')') + PsiWhiteSpace('\n\n ') + PyFunction('bar_nested') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar_nested') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testData/psi/AsyncFor.py b/python/testData/psi/AsyncFor.py new file mode 100644 index 000000000000..2eb8f7abe936 --- /dev/null +++ b/python/testData/psi/AsyncFor.py @@ -0,0 +1,7 @@ +async def f(): + async for x in xs: + pass + + +async for y in ys: + pass diff --git a/python/testData/psi/AsyncFor.txt b/python/testData/psi/AsyncFor.txt new file mode 100644 index 000000000000..abadd01f0402 --- /dev/null +++ b/python/testData/psi/AsyncFor.txt @@ -0,0 +1,54 @@ +PyFile:AsyncFor.py + PyFunction('f') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('f') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyForStatement + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PyForPart + PsiElement(Py:FOR_KEYWORD)('for') + PsiWhiteSpace(' ') + PyTargetExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiWhiteSpace(' ') + PsiElement(Py:IN_KEYWORD)('in') + PsiWhiteSpace(' ') + PyReferenceExpression: xs + PsiElement(Py:IDENTIFIER)('xs') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n\n') + PyExpressionStatement + PyReferenceExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiErrorElement:End of statement expected + + PsiWhiteSpace(' ') + PyForStatement + PyForPart + PsiElement(Py:FOR_KEYWORD)('for') + PsiWhiteSpace(' ') + PyTargetExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiWhiteSpace(' ') + PsiElement(Py:IN_KEYWORD)('in') + PsiWhiteSpace(' ') + PyReferenceExpression: ys + PsiElement(Py:IDENTIFIER)('ys') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testData/psi/AsyncWith.py b/python/testData/psi/AsyncWith.py new file mode 100644 index 000000000000..6409856e4ce6 --- /dev/null +++ b/python/testData/psi/AsyncWith.py @@ -0,0 +1,6 @@ +async def foo(): + async with x: + pass + +async with y: + pass diff --git a/python/testData/psi/AsyncWith.txt b/python/testData/psi/AsyncWith.txt new file mode 100644 index 000000000000..0313005b086d --- /dev/null +++ b/python/testData/psi/AsyncWith.txt @@ -0,0 +1,44 @@ +PyFile:AsyncWith.py + PyFunction('foo') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyWithStatement + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:WITH_KEYWORD)('with') + PsiWhiteSpace(' ') + PyWithItem + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n') + PyExpressionStatement + PyReferenceExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiErrorElement:End of statement expected + + PsiWhiteSpace(' ') + PyWithStatement + PsiElement(Py:WITH_KEYWORD)('with') + PsiWhiteSpace(' ') + PyWithItem + PyReferenceExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java index 73ab49e28659..83e625b7d3bc 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java @@ -229,6 +229,10 @@ public class PythonHighlightingTest extends PyTestCase { doTest(); } + public void testAsync() { + doTest(LanguageLevel.PYTHON35, true, true); + } + // --- private void doTest(final LanguageLevel languageLevel, final boolean checkWarnings, final boolean checkInfos) { PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), languageLevel); diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 56a4c0523f64..ecf2c39f46f3 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -486,6 +486,18 @@ public class PythonParsingTest extends ParsingTestCase { doTest(); } + public void testAsyncDef() { + doTest(LanguageLevel.PYTHON35); + } + + public void testAsyncWith() { + doTest(LanguageLevel.PYTHON35); + } + + public void testAsyncFor() { + doTest(LanguageLevel.PYTHON35); + } + public void doTest(LanguageLevel languageLevel) { LanguageLevel prev = myLanguageLevel; myLanguageLevel = languageLevel; From 4439b41170d694c84d1d0681daabc0169edc1c2f Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 12:25:10 +0300 Subject: [PATCH 02/15] Updated test data --- python/testData/highlighting/async.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/testData/highlighting/async.py b/python/testData/highlighting/async.py index 545f5ebf2c91..9749386d4d55 100644 --- a/python/testData/highlighting/async.py +++ b/python/testData/highlighting/async.py @@ -13,16 +13,16 @@ async = 1 def regular(xs): async def quux(): - async for x in xs: + async for x in xs: pass - async with xs: + async with xs: pass - async for x in xs: + async for x in xs: pass - async with xs: + async with xs: pass return async From c855959143d843fd201ce67820b994936fc3c34e Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 14:21:52 +0300 Subject: [PATCH 03/15] Parse and highlight the 'await' keyword (PY-16094) --- .../com/jetbrains/python/PyTokenTypes.java | 1 + .../com/jetbrains/python/PyElementTypes.java | 3 +- .../python/PythonTokenSetContributor.java | 2 +- .../python/parsing/ExpressionParsing.java | 17 ++++++++++- .../python/parsing/StatementParsing.java | 14 +++++++-- .../DumbAwareHighlightingAnnotator.java | 20 +++++++------ python/testData/highlighting/await.py | 5 ++++ python/testData/psi/Await.py | 5 ++++ python/testData/psi/Await.txt | 30 +++++++++++++++++++ .../python/PythonHighlightingTest.java | 4 +++ .../jetbrains/python/PythonParsingTest.java | 4 +++ 11 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 python/testData/highlighting/await.py create mode 100644 python/testData/psi/Await.py create mode 100644 python/testData/psi/Await.txt diff --git a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java index be858df92181..3e5c2037de96 100644 --- a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java +++ b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java @@ -74,6 +74,7 @@ public class PyTokenTypes { public static final PyElementType NONLOCAL_KEYWORD = new PyElementType("NONLOCAL_KEYWORD"); public static final PyElementType DEBUG_KEYWORD = new PyElementType("DEBUG_KEYWORD"); public static final PyElementType ASYNC_KEYWORD = new PyElementType("ASYNC_KEYWORD"); + public static final PyElementType AWAIT_KEYWORD = new PyElementType("AWAIT_KEYWORD", "__await__"); public static final PyElementType INTEGER_LITERAL = new PyElementType("INTEGER_LITERAL"); public static final PyElementType FLOAT_LITERAL = new PyElementType("FLOAT_LITERAL"); diff --git a/python/src/com/jetbrains/python/PyElementTypes.java b/python/src/com/jetbrains/python/PyElementTypes.java index 400e21b178c7..7d104bafb6af 100644 --- a/python/src/com/jetbrains/python/PyElementTypes.java +++ b/python/src/com/jetbrains/python/PyElementTypes.java @@ -130,7 +130,8 @@ public interface PyElementTypes { PyTokenTypes.MINUS, PyTokenTypes.MULT, PyTokenTypes.AT, PyTokenTypes.FLOORDIV, PyTokenTypes.DIV, PyTokenTypes.PERC, PyTokenTypes.EXP); - TokenSet UNARY_OPS = TokenSet.create(PyTokenTypes.NOT_KEYWORD, PyTokenTypes.PLUS, PyTokenTypes.MINUS, PyTokenTypes.TILDE); + TokenSet UNARY_OPS = TokenSet.create(PyTokenTypes.NOT_KEYWORD, PyTokenTypes.PLUS, PyTokenTypes.MINUS, PyTokenTypes.TILDE, + PyTokenTypes.AWAIT_KEYWORD); // Parts PyElementType IF_PART_IF = new PyElementType("IF_IF", PyIfPartIfImpl.class); diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 51a42589109b..ac3d95c17b30 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -73,7 +73,7 @@ public class PythonTokenSetContributor extends PythonDialectsTokenSetContributor LAMBDA_KEYWORD, NOT_KEYWORD, OR_KEYWORD, PASS_KEYWORD, PRINT_KEYWORD, RAISE_KEYWORD, RETURN_KEYWORD, TRY_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, YIELD_KEYWORD, - NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD, ASYNC_KEYWORD); + NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD, ASYNC_KEYWORD, AWAIT_KEYWORD); } @NotNull diff --git a/python/src/com/jetbrains/python/parsing/ExpressionParsing.java b/python/src/com/jetbrains/python/parsing/ExpressionParsing.java index 850f3fec2b8c..616c3d954449 100644 --- a/python/src/com/jetbrains/python/parsing/ExpressionParsing.java +++ b/python/src/com/jetbrains/python/parsing/ExpressionParsing.java @@ -915,7 +915,7 @@ public class ExpressionParsing extends Parsing { private boolean parsePowerExpression(boolean isTargetExpression) { PsiBuilder.Marker expr = myBuilder.mark(); - if (!parseMemberExpression(isTargetExpression)) { + if (!parseAwaitExpression(isTargetExpression)) { expr.drop(); return false; } @@ -933,4 +933,19 @@ public class ExpressionParsing extends Parsing { return true; } + + private boolean parseAwaitExpression(boolean isTargetExpression) { + if (atToken(PyTokenTypes.AWAIT_KEYWORD)) { + PsiBuilder.Marker expr = myBuilder.mark(); + myBuilder.advanceLexer(); + if (!parseMemberExpression(isTargetExpression)) { + myBuilder.error(message("PARSE.expected.expression")); + } + expr.done(PyElementTypes.PREFIX_EXPRESSION); + return true; + } + else { + return parseMemberExpression(isTargetExpression); + } + } } diff --git a/python/src/com/jetbrains/python/parsing/StatementParsing.java b/python/src/com/jetbrains/python/parsing/StatementParsing.java index bd6b423c033e..e5c7124c7a5d 100644 --- a/python/src/com/jetbrains/python/parsing/StatementParsing.java +++ b/python/src/com/jetbrains/python/parsing/StatementParsing.java @@ -51,6 +51,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { @NonNls protected static final String TOK_NONLOCAL = "nonlocal"; @NonNls protected static final String TOK_EXEC = "exec"; @NonNls protected static final String TOK_ASYNC = "async"; + @NonNls protected static final String TOK_AWAIT = "await"; private static final String EXPRESSION_EXPECTED = "Expression expected"; public static final String IDENTIFIER_EXPECTED = "Identifier expected"; @@ -937,9 +938,16 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (isWordAtPosition(text, start, end, TOK_NONLOCAL)) { return PyTokenTypes.NONLOCAL_KEYWORD; } - if (myContext.getLanguageLevel().isAtLeast(LanguageLevel.PYTHON35) && isWordAtPosition(text, start, end, TOK_ASYNC)) { - if (myContext.getScope().isAsync() || myBuilder.lookAhead(1) == PyTokenTypes.DEF_KEYWORD) { - return PyTokenTypes.ASYNC_KEYWORD; + if (myContext.getLanguageLevel().isAtLeast(LanguageLevel.PYTHON35)) { + if (isWordAtPosition(text, start, end, TOK_ASYNC)) { + if (myContext.getScope().isAsync() || myBuilder.lookAhead(1) == PyTokenTypes.DEF_KEYWORD) { + return PyTokenTypes.ASYNC_KEYWORD; + } + } + if (isWordAtPosition(text, start, end, TOK_AWAIT)) { + if (myContext.getScope().isAsync()) { + return PyTokenTypes.AWAIT_KEYWORD; + } } } } diff --git a/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java index 717d59f8e37d..b90f8697c6a2 100644 --- a/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java +++ b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java @@ -22,10 +22,7 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.highlighting.PyHighlighter; -import com.jetbrains.python.psi.PyFile; -import com.jetbrains.python.psi.PyForStatement; -import com.jetbrains.python.psi.PyFunction; -import com.jetbrains.python.psi.PyWithStatement; +import com.jetbrains.python.psi.*; import org.jetbrains.annotations.NotNull; /** @@ -34,17 +31,22 @@ import org.jetbrains.annotations.NotNull; public class DumbAwareHighlightingAnnotator extends PyAnnotator implements HighlightRangeExtension { @Override public void visitPyFunction(PyFunction node) { - highlightAsyncKeyword(node); + highlightKeyword(node, PyTokenTypes.ASYNC_KEYWORD); } @Override public void visitPyForStatement(PyForStatement node) { - highlightAsyncKeyword(node); + highlightKeyword(node, PyTokenTypes.ASYNC_KEYWORD); } @Override public void visitPyWithStatement(PyWithStatement node) { - highlightAsyncKeyword(node); + highlightKeyword(node, PyTokenTypes.ASYNC_KEYWORD); + } + + @Override + public void visitPyPrefixExpression(PyPrefixExpression node) { + highlightKeyword(node, PyTokenTypes.AWAIT_KEYWORD); } @Override @@ -52,8 +54,8 @@ public class DumbAwareHighlightingAnnotator extends PyAnnotator implements Highl return file instanceof PyFile; } - private void highlightAsyncKeyword(@NotNull PsiElement node) { - final ASTNode asyncNode = node.getNode().findChildByType(PyTokenTypes.ASYNC_KEYWORD); + private void highlightKeyword(@NotNull PsiElement node, @NotNull PyElementType elementType) { + final ASTNode asyncNode = node.getNode().findChildByType(elementType); if (asyncNode != null) { final Annotation annotation = getHolder().createInfoAnnotation(asyncNode, null); annotation.setTextAttributes(PyHighlighter.PY_KEYWORD); diff --git a/python/testData/highlighting/await.py b/python/testData/highlighting/await.py new file mode 100644 index 000000000000..ec8e80b683b2 --- /dev/null +++ b/python/testData/highlighting/await.py @@ -0,0 +1,5 @@ +async def foo(): + await x + + +await = 0 diff --git a/python/testData/psi/Await.py b/python/testData/psi/Await.py new file mode 100644 index 000000000000..58bfdcaaa1dd --- /dev/null +++ b/python/testData/psi/Await.py @@ -0,0 +1,5 @@ +async def f(x): + await x + + +await = 1 diff --git a/python/testData/psi/Await.txt b/python/testData/psi/Await.txt new file mode 100644 index 000000000000..9b5ee093ceb9 --- /dev/null +++ b/python/testData/psi/Await.txt @@ -0,0 +1,30 @@ +PyFile:Await.py + PyFunction('f') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('f') + PyParameterList + PsiElement(Py:LPAR)('(') + PyNamedParameter('x') + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyExpressionStatement + PyPrefixExpression + PsiElement(Py:AWAIT_KEYWORD)('await') + PsiWhiteSpace(' ') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiWhiteSpace('\n\n\n') + PyAssignmentStatement + PyTargetExpression: await + PsiElement(Py:IDENTIFIER)('await') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java index 83e625b7d3bc..3095bda21611 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java @@ -233,6 +233,10 @@ public class PythonHighlightingTest extends PyTestCase { doTest(LanguageLevel.PYTHON35, true, true); } + public void testAwait() { + doTest(LanguageLevel.PYTHON35, true, true); + } + // --- private void doTest(final LanguageLevel languageLevel, final boolean checkWarnings, final boolean checkInfos) { PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), languageLevel); diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index ecf2c39f46f3..99172dce2e48 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -498,6 +498,10 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON35); } + public void testAwait() { + doTest(LanguageLevel.PYTHON35); + } + public void doTest(LanguageLevel languageLevel) { LanguageLevel prev = myLanguageLevel; myLanguageLevel = languageLevel; From c8a8585c36f13ca121ede20cdf12986410c51795 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 14:22:17 +0300 Subject: [PATCH 04/15] Code completion for 'async' and 'await' (PY-16094) --- .../src/com/jetbrains/python/PyNames.java | 2 + .../com/jetbrains/python/psi/PyFunction.java | 2 + .../python/psi/stubs/PyFunctionStub.java | 1 + .../PyKeywordCompletionContributor.java | 57 +++++++++++++++++-- .../python/psi/PyFileElementType.java | 2 +- .../python/psi/impl/PyFunctionImpl.java | 9 +++ .../psi/impl/stubs/PyFunctionElementType.java | 8 ++- .../psi/impl/stubs/PyFunctionStubImpl.java | 9 ++- python/testData/completion/async.after.py | 1 + python/testData/completion/async.py | 1 + python/testData/completion/await.after.py | 2 + python/testData/completion/await.py | 2 + .../jetbrains/python/Py3CompletionTest.java | 18 ++++++ 13 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 python/testData/completion/async.after.py create mode 100644 python/testData/completion/async.py create mode 100644 python/testData/completion/await.after.py create mode 100644 python/testData/completion/await.py diff --git a/python/psi-api/src/com/jetbrains/python/PyNames.java b/python/psi-api/src/com/jetbrains/python/PyNames.java index f916bcf78264..dd799b45f763 100644 --- a/python/psi-api/src/com/jetbrains/python/PyNames.java +++ b/python/psi-api/src/com/jetbrains/python/PyNames.java @@ -450,6 +450,8 @@ public class PyNames { public static final String IN = "in"; public static final String NOT = "not"; public static final String LAMBDA = "lambda"; + public static final String ASYNC = "async"; + public static final String AWAIT = "await"; /** * Contains keywords as of CPython 2.5. diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyFunction.java b/python/psi-api/src/com/jetbrains/python/psi/PyFunction.java index afd1bfcb543c..01cd6d5234d6 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyFunction.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyFunction.java @@ -79,6 +79,8 @@ extends @Nullable Modifier getModifier(); + boolean isAsync(); + /** * Flags that mark common alterations of a function: decoration by and wrapping in classmethod() and staticmethod(). */ diff --git a/python/psi-api/src/com/jetbrains/python/psi/stubs/PyFunctionStub.java b/python/psi-api/src/com/jetbrains/python/psi/stubs/PyFunctionStub.java index 78cd774e20b9..1b4bcb7bacab 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/stubs/PyFunctionStub.java +++ b/python/psi-api/src/com/jetbrains/python/psi/stubs/PyFunctionStub.java @@ -21,4 +21,5 @@ import com.jetbrains.python.psi.PyFunction; public interface PyFunctionStub extends NamedStub { String getDocString(); String getDeprecationMessage(); + boolean isAsync(); } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/codeInsight/completion/PyKeywordCompletionContributor.java b/python/src/com/jetbrains/python/codeInsight/completion/PyKeywordCompletionContributor.java index 387fc05830aa..5873272ae500 100644 --- a/python/src/com/jetbrains/python/codeInsight/completion/PyKeywordCompletionContributor.java +++ b/python/src/com/jetbrains/python/codeInsight/completion/PyKeywordCompletionContributor.java @@ -34,6 +34,8 @@ import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.PythonLanguage; import com.jetbrains.python.codeInsight.PyUnindentingInsertHandler; +import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; +import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; import com.jetbrains.python.documentation.doctest.PyDocstringFile; import com.jetbrains.python.psi.*; import org.jetbrains.annotations.NonNls; @@ -182,13 +184,19 @@ public class PyKeywordCompletionContributor extends CompletionContributor { } } - private static class Py3kFilter implements ElementFilter { + private static class LanguageLevelAtLeastFilter implements ElementFilter { + @NotNull private final LanguageLevel myLevel; + + public LanguageLevelAtLeastFilter(@NotNull LanguageLevel level) { + myLevel = level; + } + public boolean isAcceptable(Object element, PsiElement context) { if (!(element instanceof PsiElement)) { return false; } final PsiFile containingFile = ((PsiElement)element).getContainingFile(); - return containingFile instanceof PyFile && ((PyFile)containingFile).getLanguageLevel().isPy3K(); + return containingFile instanceof PyFile && ((PyFile)containingFile).getLanguageLevel().isAtLeast(myLevel); } public boolean isClassAcceptable(Class hintClass) { @@ -196,6 +204,7 @@ public class PyKeywordCompletionContributor extends CompletionContributor { } } + private static class NotParameterOrDefaultValue implements ElementFilter { @Override @@ -359,7 +368,8 @@ public class PyKeywordCompletionContributor extends CompletionContributor { )); */ - private static final FilterPattern PY3K = new FilterPattern(new Py3kFilter()); + private static final FilterPattern PY3K = new FilterPattern(new PyKeywordCompletionContributor.LanguageLevelAtLeastFilter(LanguageLevel.PYTHON30)); + private static final FilterPattern PY35 = new FilterPattern(new LanguageLevelAtLeastFilter(LanguageLevel.PYTHON35)); // ====== @@ -477,6 +487,28 @@ public class PyKeywordCompletionContributor extends CompletionContributor { , new PyKeywordCompletionProvider(PyNames.NONLOCAL) ); + + extend(CompletionType.BASIC, + psiElement() + .withLanguage(PythonLanguage.getInstance()) + .and(PY35) + .andNot(AFTER_QUALIFIER) + .with(new PatternCondition("insideAsyncDef") { + @Override + public boolean accepts(@NotNull PsiElement element, ProcessingContext context) { + final ScopeOwner owner = ScopeUtil.getScopeOwner(element); + return owner instanceof PyFunction && ((PyFunction)owner).isAsync(); + } + }) + .andOr(IN_BEGIN_STMT, + psiElement() + .inside(false, psiElement(PyAssignmentStatement.class), psiElement(PyTargetExpression.class)) + .afterLeaf(psiElement().withElementType(PyTokenTypes.EQ)), + psiElement() + .inside(false, psiElement(PyAugAssignmentStatement.class), psiElement(PyTargetExpression.class)) + .afterLeaf(psiElement().withElementType(PyTokenTypes.AUG_ASSIGN_OPERATIONS)), + psiElement().inside(true, psiElement(PyParenthesizedExpression.class))), + new PyKeywordCompletionProvider(PyNames.AWAIT)); } private void addWithinIf() { @@ -578,8 +610,23 @@ public class PyKeywordCompletionContributor extends CompletionContributor { .andNot(AFTER_QUALIFIER) .andNot(IN_FUNCTION_HEADER) , - new PyKeywordCompletionProvider(TailType.NONE, PyNames.TRUE, PyNames.FALSE, PyNames.NONE) - ); + new PyKeywordCompletionProvider(TailType.NONE, PyNames.TRUE, PyNames.FALSE, PyNames.NONE)); + extend(CompletionType.BASIC, + psiElement() + .withLanguage(PythonLanguage.getInstance()) + .and(PY35) + .andNot(IN_COMMENT) + .andNot(IN_IMPORT_STMT) + .andNot(IN_PARAM_LIST) + .andNot(AFTER_QUALIFIER) + .andNot(IN_STRING_LITERAL), + new PyKeywordCompletionProvider(PyNames.ASYNC)); + extend(CompletionType.BASIC, + psiElement() + .withLanguage(PythonLanguage.getInstance()) + .and(PY35) + .afterLeaf(psiElement().withElementType(PyTokenTypes.IDENTIFIER).withText(PyNames.ASYNC)), + new PyKeywordCompletionProvider(PyNames.DEF, PyNames.WITH, PyNames.FOR)); } private void addAs() { diff --git a/python/src/com/jetbrains/python/psi/PyFileElementType.java b/python/src/com/jetbrains/python/psi/PyFileElementType.java index d646679ffd18..7e75bc364657 100644 --- a/python/src/com/jetbrains/python/psi/PyFileElementType.java +++ b/python/src/com/jetbrains/python/psi/PyFileElementType.java @@ -63,7 +63,7 @@ public class PyFileElementType extends IStubFileElementType { @Override public int getStubVersion() { // Don't forget to update versions of indexes that use the updated stub-based elements - return 50; + return 51; } @Nullable diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index 66e9248e0761..bab647903f24 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -638,6 +638,15 @@ public class PyFunctionImpl extends PyBaseElementImpl implements return null; } + @Override + public boolean isAsync() { + final PyFunctionStub stub = getStub(); + if (stub != null) { + return stub.isAsync(); + } + return getNode().findChildByType(PyTokenTypes.ASYNC_KEYWORD) != null; + } + @Nullable private Modifier getWrappersFromStub() { final StubElement parentStub = getStub().getParentStub(); diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionElementType.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionElementType.java index aca549a72cad..e1b1ec3158ee 100644 --- a/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionElementType.java +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionElementType.java @@ -56,7 +56,8 @@ public class PyFunctionElementType extends PyStubElementType 0 ? docString : null, deprecationMessage, parentStub, getStubElementType()); + final boolean isAsync = dataStream.readBoolean(); + return new PyFunctionStubImpl(name, docString.length() > 0 ? docString : null, deprecationMessage, isAsync, parentStub, + getStubElementType()); } public void indexStub(@NotNull final PyFunctionStub stub, @NotNull final IndexSink sink) { diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionStubImpl.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionStubImpl.java index e1665595dada..524798a6c199 100644 --- a/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionStubImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyFunctionStubImpl.java @@ -27,13 +27,15 @@ public class PyFunctionStubImpl extends StubBase implements PyFuncti private final String myName; private final String myDocString; private final StringRef myDeprecationMessage; + private final boolean myAsync; - public PyFunctionStubImpl(final String name, final String docString, @Nullable final StringRef deprecationMessage, + public PyFunctionStubImpl(final String name, final String docString, @Nullable final StringRef deprecationMessage, boolean isAsync, final StubElement parent, IStubElementType stubElementType) { super(parent, stubElementType); myName = name; myDocString = docString; myDeprecationMessage = deprecationMessage; + myAsync = isAsync; } public String getName() { @@ -49,6 +51,11 @@ public class PyFunctionStubImpl extends StubBase implements PyFuncti return myDeprecationMessage == null ? null : myDeprecationMessage.getString(); } + @Override + public boolean isAsync() { + return myAsync; + } + @Override public String toString() { return "PyFunctionStub(" + myName + ")"; diff --git a/python/testData/completion/async.after.py b/python/testData/completion/async.after.py new file mode 100644 index 000000000000..500848fbe1b1 --- /dev/null +++ b/python/testData/completion/async.after.py @@ -0,0 +1 @@ +async # Comment diff --git a/python/testData/completion/async.py b/python/testData/completion/async.py new file mode 100644 index 000000000000..ff8fb3181275 --- /dev/null +++ b/python/testData/completion/async.py @@ -0,0 +1 @@ +asy # Comment diff --git a/python/testData/completion/await.after.py b/python/testData/completion/await.after.py new file mode 100644 index 000000000000..2de7a7d8d0cd --- /dev/null +++ b/python/testData/completion/await.after.py @@ -0,0 +1,2 @@ +async def foo(): + await # comment diff --git a/python/testData/completion/await.py b/python/testData/completion/await.py new file mode 100644 index 000000000000..110aec23d3cf --- /dev/null +++ b/python/testData/completion/await.py @@ -0,0 +1,2 @@ +async def foo(): + awa # comment diff --git a/python/testSrc/com/jetbrains/python/Py3CompletionTest.java b/python/testSrc/com/jetbrains/python/Py3CompletionTest.java index 64bdf670135e..e20f0f403768 100644 --- a/python/testSrc/com/jetbrains/python/Py3CompletionTest.java +++ b/python/testSrc/com/jetbrains/python/Py3CompletionTest.java @@ -157,6 +157,24 @@ public class Py3CompletionTest extends PyTestCase { doTest(); } + public void testAsync() { + runWithLanguageLevel(LanguageLevel.PYTHON35, new Runnable() { + @Override + public void run() { + doTest(); + } + }); + } + + public void testAwait() { + runWithLanguageLevel(LanguageLevel.PYTHON35, new Runnable() { + @Override + public void run() { + doTest(); + } + }); + } + @Override protected String getTestDataPath() { return super.getTestDataPath() + "/completion"; From e2d75292f0110bf261c80bf15e2f7335bcb80e86 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 15:03:39 +0300 Subject: [PATCH 05/15] Annotate 'yield' inside async functions as errors (PY-16094) --- .../src/com/jetbrains/python/validation/ReturnAnnotator.java | 5 ++++- python/testData/highlighting/yieldInsideAsyncDef.py | 5 +++++ .../testSrc/com/jetbrains/python/PythonHighlightingTest.java | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 python/testData/highlighting/yieldInsideAsyncDef.py diff --git a/python/src/com/jetbrains/python/validation/ReturnAnnotator.java b/python/src/com/jetbrains/python/validation/ReturnAnnotator.java index e0057e02f192..f665bcf8893a 100644 --- a/python/src/com/jetbrains/python/validation/ReturnAnnotator.java +++ b/python/src/com/jetbrains/python/validation/ReturnAnnotator.java @@ -21,7 +21,7 @@ import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; import com.jetbrains.python.psi.*; /** - * Highlights incorrect return statements: 'return' and 'yield' outside functions, returning values from generators. + * Highlights incorrect return statements: 'return' and 'yield' outside functions, 'yield' inside async functions. */ public class ReturnAnnotator extends PyAnnotator { public void visitPyReturnStatement(final PyReturnStatement node) { @@ -36,5 +36,8 @@ public class ReturnAnnotator extends PyAnnotator { if (!(owner instanceof PyFunction || owner instanceof PyLambdaExpression)) { getHolder().createErrorAnnotation(node, "'yield' outside of function"); } + if (owner instanceof PyFunction && ((PyFunction)owner).isAsync()) { + getHolder().createErrorAnnotation(node, "'yield' inside async function"); + } } } diff --git a/python/testData/highlighting/yieldInsideAsyncDef.py b/python/testData/highlighting/yieldInsideAsyncDef.py new file mode 100644 index 000000000000..0b4b8f47aac4 --- /dev/null +++ b/python/testData/highlighting/yieldInsideAsyncDef.py @@ -0,0 +1,5 @@ +async def foo(x): + await x + yield x + yield from x + return x diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java index 3095bda21611..d0d4e894ea23 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java @@ -237,6 +237,10 @@ public class PythonHighlightingTest extends PyTestCase { doTest(LanguageLevel.PYTHON35, true, true); } + public void testYieldInsideAsyncDef() { + doTest(LanguageLevel.PYTHON35, false, false); + } + // --- private void doTest(final LanguageLevel languageLevel, final boolean checkWarnings, final boolean checkInfos) { PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), languageLevel); From 796c6a1fc23c3e75bf0a13c4630e1605564b33e9 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 16:39:13 +0300 Subject: [PATCH 06/15] Compatibility inpsection for async-await (PY-16094) --- .../PyCompatibilityInspection.java | 2 +- .../validation/CompatibilityVisitor.java | 38 +++++++++++++++++++ .../DumbAwareHighlightingAnnotator.java | 6 +-- .../PyCompatibilityInspection/asyncAwait.py | 7 ++++ .../PyCompatibilityInspectionTest.java | 4 ++ 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 python/testData/inspections/PyCompatibilityInspection/asyncAwait.py diff --git a/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java b/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java index 85af8719c7a1..a93813437f2b 100644 --- a/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyCompatibilityInspection.java @@ -161,7 +161,7 @@ public class PyCompatibilityInspection extends PyInspection { if (element.getTextLength() == 0) { return; } - range = TextRange.create(range.getStartOffset() - element.getTextOffset(), range.getEndOffset() - element.getTextOffset()); + range = range.shiftRight(-element.getTextRange().getStartOffset()); if (quickFix != null) myHolder.registerProblem(element, range, message, quickFix); else diff --git a/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java b/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java index fb6d344475c7..ddc12854d316 100644 --- a/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java +++ b/python/src/com/jetbrains/python/validation/CompatibilityVisitor.java @@ -383,6 +383,13 @@ public abstract class CompatibilityVisitor extends PyAnnotator { for (PyWithItem item : problemItems) { registerProblem(item, message.toString()); } + checkAsyncKeyword(node); + } + + @Override + public void visitPyForStatement(PyForStatement node) { + super.visitPyForStatement(node); + checkAsyncKeyword(node); } @Override @@ -507,6 +514,25 @@ public abstract class CompatibilityVisitor extends PyAnnotator { len, node, null); } + @Override + public void visitPyFunction(PyFunction node) { + super.visitPyFunction(node); + checkAsyncKeyword(node); + } + + @Override + public void visitPyPrefixExpression(PyPrefixExpression node) { + super.visitPyPrefixExpression(node); + if (node.getOperator() == PyTokenTypes.AWAIT_KEYWORD) { + for (LanguageLevel level : myVersionsToProcess) { + if (level.isOlderThan(LanguageLevel.PYTHON35)) { + registerProblem(node, "Python versions < 3.5 do not support this syntax"); + break; + } + } + } + } + @Override public void visitPyYieldExpression(PyYieldExpression node) { super.visitPyYieldExpression(node); @@ -571,6 +597,18 @@ public abstract class CompatibilityVisitor extends PyAnnotator { } } + private void checkAsyncKeyword(PsiElement node) { + final ASTNode asyncNode = node.getNode().findChildByType(PyTokenTypes.ASYNC_KEYWORD); + if (asyncNode != null) { + for (LanguageLevel level : myVersionsToProcess) { + if (level.isOlderThan(LanguageLevel.PYTHON35)) { + registerProblem(node, asyncNode.getTextRange(), "Python versions < 3.5 do not support this syntax", null, true); + break; + } + } + } + } + private static class YieldVisitor extends PyElementVisitor { private boolean _haveYield = false; diff --git a/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java index b90f8697c6a2..e74034216dae 100644 --- a/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java +++ b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java @@ -55,9 +55,9 @@ public class DumbAwareHighlightingAnnotator extends PyAnnotator implements Highl } private void highlightKeyword(@NotNull PsiElement node, @NotNull PyElementType elementType) { - final ASTNode asyncNode = node.getNode().findChildByType(elementType); - if (asyncNode != null) { - final Annotation annotation = getHolder().createInfoAnnotation(asyncNode, null); + final ASTNode astNode = node.getNode().findChildByType(elementType); + if (astNode != null) { + final Annotation annotation = getHolder().createInfoAnnotation(astNode, null); annotation.setTextAttributes(PyHighlighter.PY_KEYWORD); } } diff --git a/python/testData/inspections/PyCompatibilityInspection/asyncAwait.py b/python/testData/inspections/PyCompatibilityInspection/asyncAwait.py new file mode 100644 index 000000000000..222cf799090f --- /dev/null +++ b/python/testData/inspections/PyCompatibilityInspection/asyncAwait.py @@ -0,0 +1,7 @@ +async def foo(x): + async with x: + y = await x + if await y: + return await z + async for y in x: + pass diff --git a/python/testSrc/com/jetbrains/python/inspections/PyCompatibilityInspectionTest.java b/python/testSrc/com/jetbrains/python/inspections/PyCompatibilityInspectionTest.java index b941defb2b16..49e9acb6929f 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyCompatibilityInspectionTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyCompatibilityInspectionTest.java @@ -155,6 +155,10 @@ public class PyCompatibilityInspectionTest extends PyTestCase { doTest(LanguageLevel.PYTHON35); } + public void testAsyncAwait() { + doTest(LanguageLevel.PYTHON35); + } + private void doTest(@NotNull LanguageLevel level) { runWithLanguageLevel(level, new Runnable() { @Override From 47b43928c90cdd614e4a7af4adfb6b08abc99115 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 16:48:59 +0300 Subject: [PATCH 07/15] Moved statement effect inspection tests into a separate class --- .../{test.py => basic.py} | 0 .../python/PythonInspectionsTest.java | 4 --- .../PyStatementEffectInspectionTest.java | 34 +++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) rename python/testData/inspections/PyStatementEffectInspection/{test.py => basic.py} (100%) create mode 100644 python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java diff --git a/python/testData/inspections/PyStatementEffectInspection/test.py b/python/testData/inspections/PyStatementEffectInspection/basic.py similarity index 100% rename from python/testData/inspections/PyStatementEffectInspection/test.py rename to python/testData/inspections/PyStatementEffectInspection/basic.py diff --git a/python/testSrc/com/jetbrains/python/PythonInspectionsTest.java b/python/testSrc/com/jetbrains/python/PythonInspectionsTest.java index 4665c787317b..8fc2d8b2eca9 100644 --- a/python/testSrc/com/jetbrains/python/PythonInspectionsTest.java +++ b/python/testSrc/com/jetbrains/python/PythonInspectionsTest.java @@ -159,10 +159,6 @@ public class PythonInspectionsTest extends PyTestCase { } } - public void testPyStatementEffectInspection() { - doHighlightingTest(PyStatementEffectInspection.class, LanguageLevel.PYTHON26); - } - public void testPySimplifyBooleanCheckInspection() { doHighlightingTest(PySimplifyBooleanCheckInspection.class, LanguageLevel.PYTHON26); } diff --git a/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java b/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java new file mode 100644 index 000000000000..92588bd40bea --- /dev/null +++ b/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.inspections; + +import com.jetbrains.python.fixtures.PyInspectionTestCase; +import org.jetbrains.annotations.NotNull; + +/** + * @author vlan + */ +public class PyStatementEffectInspectionTest extends PyInspectionTestCase { + public void testBasic() { + doTest(); + } + + @NotNull + @Override + protected Class getInspectionClass() { + return PyStatementEffectInspection.class; + } +} From ef0fddb5892707ac0b0a8d95e5050b98ecdbe692 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 17:13:25 +0300 Subject: [PATCH 08/15] Don't highlight 'await' expression statements as having no effect (PY-16094) --- .../inspections/PyStatementEffectInspection.java | 5 +++++ .../PyStatementEffectInspection/await.py | 8 ++++++++ .../PyStatementEffectInspectionTest.java | 14 ++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 python/testData/inspections/PyStatementEffectInspection/await.py diff --git a/python/src/com/jetbrains/python/inspections/PyStatementEffectInspection.java b/python/src/com/jetbrains/python/inspections/PyStatementEffectInspection.java index f5a087ffa21b..b0d8896c48e8 100644 --- a/python/src/com/jetbrains/python/inspections/PyStatementEffectInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyStatementEffectInspection.java @@ -21,6 +21,7 @@ import com.intellij.psi.PsiElementVisitor; import com.intellij.psi.ResolveResult; import com.intellij.psi.util.PsiTreeUtil; import com.jetbrains.python.PyBundle; +import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.inspections.quickfix.StatementEffectFunctionCallQuickFix; import com.jetbrains.python.inspections.quickfix.StatementEffectIntroduceVariableQuickFix; import com.jetbrains.python.psi.*; @@ -149,6 +150,10 @@ public class PyStatementEffectInspection extends PyInspection { } } } + else if (expression instanceof PyPrefixExpression) { + final PyPrefixExpression prefixExpr = (PyPrefixExpression)expression; + return prefixExpr.getOperator() == PyTokenTypes.AWAIT_KEYWORD; + } return false; } } diff --git a/python/testData/inspections/PyStatementEffectInspection/await.py b/python/testData/inspections/PyStatementEffectInspection/await.py new file mode 100644 index 000000000000..213d8c4a2eb0 --- /dev/null +++ b/python/testData/inspections/PyStatementEffectInspection/await.py @@ -0,0 +1,8 @@ +async def f(x): + y = await x + await x + if await x: + pass + f(await x) + x + return await x diff --git a/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java b/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java index 92588bd40bea..c61ddd5cf081 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyStatementEffectInspectionTest.java @@ -16,6 +16,7 @@ package com.jetbrains.python.inspections; import com.jetbrains.python.fixtures.PyInspectionTestCase; +import com.jetbrains.python.psi.LanguageLevel; import org.jetbrains.annotations.NotNull; /** @@ -26,6 +27,19 @@ public class PyStatementEffectInspectionTest extends PyInspectionTestCase { doTest(); } + public void testAwait() { + doTest(LanguageLevel.PYTHON35); + } + + private void doTest(@NotNull LanguageLevel level) { + runWithLanguageLevel(level, new Runnable() { + @Override + public void run() { + doTest(); + } + }); + } + @NotNull @Override protected Class getInspectionClass() { From caf85e8e7194fcbd6f758a26a2743b61487fb3d6 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 8 Sep 2015 17:44:21 +0300 Subject: [PATCH 09/15] Fixed parsing decorated async functions and 'await' expressions inside them (PY-16094) --- .../python/parsing/FunctionParsing.java | 12 ++++ python/testData/psi/DecoratedAsyncDef.py | 9 +++ python/testData/psi/DecoratedAsyncDef.txt | 69 +++++++++++++++++++ .../jetbrains/python/PythonParsingTest.java | 4 ++ 4 files changed, 94 insertions(+) create mode 100644 python/testData/psi/DecoratedAsyncDef.py create mode 100644 python/testData/psi/DecoratedAsyncDef.txt diff --git a/python/src/com/jetbrains/python/parsing/FunctionParsing.java b/python/src/com/jetbrains/python/parsing/FunctionParsing.java index 2c8b7dff584b..6ea9a38558de 100644 --- a/python/src/com/jetbrains/python/parsing/FunctionParsing.java +++ b/python/src/com/jetbrains/python/parsing/FunctionParsing.java @@ -99,6 +99,18 @@ public class FunctionParsing extends Parsing { } protected void parseDeclarationAfterDecorator(PsiBuilder.Marker endMarker) { + if (myBuilder.getTokenType() == PyTokenTypes.ASYNC_KEYWORD) { + myBuilder.advanceLexer(); + myContext.pushScope(myContext.getScope().withAsync()); + parseSyncDeclarationAfterDecorator(endMarker); + myContext.popScope(); + } + else { + parseSyncDeclarationAfterDecorator(endMarker); + } + } + + private void parseSyncDeclarationAfterDecorator(PsiBuilder.Marker endMarker) { if (myBuilder.getTokenType() == PyTokenTypes.DEF_KEYWORD) { parseFunctionInnards(endMarker); } diff --git a/python/testData/psi/DecoratedAsyncDef.py b/python/testData/psi/DecoratedAsyncDef.py new file mode 100644 index 000000000000..c2bfe7f6c0ba --- /dev/null +++ b/python/testData/psi/DecoratedAsyncDef.py @@ -0,0 +1,9 @@ +@foo +async def bar(): + await x + return 0 + + +@baz(x, y) +async def quux(): + return await x diff --git a/python/testData/psi/DecoratedAsyncDef.txt b/python/testData/psi/DecoratedAsyncDef.txt new file mode 100644 index 000000000000..c8b35f5b486e --- /dev/null +++ b/python/testData/psi/DecoratedAsyncDef.txt @@ -0,0 +1,69 @@ +PyFile:DecoratedAsyncDef.py + PyFunction('bar') + PyDecoratorList + PyDecorator: @foo + PsiElement(Py:AT)('@') + PyReferenceExpression: foo + PsiElement(Py:IDENTIFIER)('foo') + PyArgumentList + + PsiWhiteSpace('\n') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyExpressionStatement + PyPrefixExpression + PsiElement(Py:AWAIT_KEYWORD)('await') + PsiWhiteSpace(' ') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiWhiteSpace('\n ') + PyReturnStatement + PsiElement(Py:RETURN_KEYWORD)('return') + PsiWhiteSpace(' ') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('0') + PsiWhiteSpace('\n\n\n') + PyFunction('quux') + PyDecoratorList + PyDecorator: @baz + PsiElement(Py:AT)('@') + PyReferenceExpression: baz + PsiElement(Py:IDENTIFIER)('baz') + PyArgumentList + PsiElement(Py:LPAR)('(') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyReferenceExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:RPAR)(')') + PsiWhiteSpace('\n') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('quux') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyReturnStatement + PsiElement(Py:RETURN_KEYWORD)('return') + PsiWhiteSpace(' ') + PyPrefixExpression + PsiElement(Py:AWAIT_KEYWORD)('await') + PsiWhiteSpace(' ') + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 99172dce2e48..a96a6d2c5706 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -502,6 +502,10 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON35); } + public void testDecoratedAsyncDef() { + doTest(LanguageLevel.PYTHON35); + } + public void doTest(LanguageLevel languageLevel) { LanguageLevel prev = myLanguageLevel; myLanguageLevel = languageLevel; From aad613af6b0f86bbd9f048e24a480aca2870e79e Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Fri, 11 Sep 2015 19:37:50 +0300 Subject: [PATCH 10/15] Update user-skeletons path in SDK, fixed a bug with removeRoots() instead of addRoot() --- .../python/sdk/PythonSdkUpdater.java | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java index 3b2f12b5a63a..44bf7b800ce5 100644 --- a/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java +++ b/python/src/com/jetbrains/python/sdk/PythonSdkUpdater.java @@ -194,7 +194,8 @@ public class PythonSdkUpdater implements StartupActivity { addNewSysPathEntries(sdkUpdater, sysPath); removeSourceRoots(sdkUpdater); removeDuplicateClassRoots(sdkUpdater); - updateSkeletonsPath(sdkUpdater); + updateBinarySkeletonsPath(sdkUpdater); + updateUserSkeletonsPath(sdkUpdater); } /** @@ -251,29 +252,42 @@ public class PythonSdkUpdater implements StartupActivity { return false; } + /** + * Updates user skeletons path in the Python SDK table. + */ + private static void updateUserSkeletonsPath(@NotNull PySdkUpdater sdkUpdater) { + updateSkeletonsPath(sdkUpdater, PyUserSkeletonsUtil.getUserSkeletonsDirectory(), PyUserSkeletonsUtil.USER_SKELETONS_DIR, + "User skeletons"); + } + /** * Updates binary skeletons path in the Python SDK table. */ - private static void updateSkeletonsPath(@NotNull PySdkUpdater sdkUpdater) { + private static void updateBinarySkeletonsPath(@NotNull PySdkUpdater sdkUpdater) { final String skeletonsPath = PythonSdkType.getSkeletonsPath(PathManager.getSystemPath(), sdkUpdater.getHomePath()); if (skeletonsPath != null) { final VirtualFile skeletonsDir = StandardFileSystems.local().refreshAndFindFileByPath(skeletonsPath); if (skeletonsDir != null) { - LOG.info("Binary skeletons directory for SDK \"" + sdkUpdater.getSdk().getName() + "\" (" + sdkUpdater.getHomePath() + "): " + skeletonsDir.getPath()); - final List sourceRoots = Arrays.asList(sdkUpdater.getSdk().getRootProvider().getFiles(OrderRootType.CLASSES)); - boolean skeletonsDirFound = false; - for (final VirtualFile root : sourceRoots) { - if (root.equals(skeletonsDir)) { - skeletonsDirFound = true; - } - if (PythonSdkType.isSkeletonsPath(root.getPath()) && !skeletonsDirFound) { - sdkUpdater.addRoot(root, OrderRootType.CLASSES); - } - } - if (!skeletonsDirFound) { - sdkUpdater.addRoot(skeletonsDir, OrderRootType.CLASSES); + updateSkeletonsPath(sdkUpdater, skeletonsDir, PythonSdkType.SKELETON_DIR_NAME, "Binary skeletons"); + } + } + } + + private static void updateSkeletonsPath(@NotNull PySdkUpdater sdkUpdater, + @Nullable VirtualFile skeletonsDir, + @NotNull String skeletonsDirPattern, + @NotNull String skeletonsTitle) { + if (skeletonsDir != null) { + LOG.info(skeletonsTitle + " directory for SDK \"" + sdkUpdater.getSdk().getName() + "\" (" + sdkUpdater.getHomePath() + "): " + + skeletonsDir.getPath()); + final List sourceRoots = Arrays.asList(sdkUpdater.getSdk().getRootProvider().getFiles(OrderRootType.CLASSES)); + sdkUpdater.removeRoots(OrderRootType.CLASSES); + for (final VirtualFile root : sourceRoots) { + if (!root.getPath().contains(skeletonsDirPattern)) { + sdkUpdater.addRoot(root, OrderRootType.CLASSES); } } + sdkUpdater.addRoot(skeletonsDir, OrderRootType.CLASSES); } } From de542bb7a8a7e4d55150d7c0fe9036d7edea75ce Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Fri, 11 Sep 2015 20:23:30 +0300 Subject: [PATCH 11/15] Added __coroutine fake type to the builtins binary skeleton for Python 3.5 --- .../pycharm_generator_utils/constants.py | 2 +- .../module_redeclarator.py | 2 + .../pycharm_generator_utils/util_methods.py | 33 + .../helpers/python-skeletons/__builtin__.py | 24 +- python/helpers/python-skeletons/builtins.py | 24 +- python/helpers/required_gen_version | 2 +- .../MockSdk3.4/python_stubs/builtins.py | 1136 ++++++++--------- 7 files changed, 583 insertions(+), 640 deletions(-) diff --git a/python/helpers/pycharm_generator_utils/constants.py b/python/helpers/pycharm_generator_utils/constants.py index fa3fa71a0e9a..1839fba532eb 100644 --- a/python/helpers/pycharm_generator_utils/constants.py +++ b/python/helpers/pycharm_generator_utils/constants.py @@ -6,7 +6,7 @@ import string import time # !!! Don't forget to update VERSION and required_gen_version if necessary !!! -VERSION = "1.136" +VERSION = "1.137" OUT_ENCODING = 'utf-8' diff --git a/python/helpers/pycharm_generator_utils/module_redeclarator.py b/python/helpers/pycharm_generator_utils/module_redeclarator.py index e6bf988333a7..ec7346f4e1de 100644 --- a/python/helpers/pycharm_generator_utils/module_redeclarator.py +++ b/python/helpers/pycharm_generator_utils/module_redeclarator.py @@ -1014,6 +1014,8 @@ class ModuleRedeclarator(object): self.classes_buf.out(0, txt) txt = create_method() self.classes_buf.out(0, txt) + txt = create_coroutine() + self.classes_buf.out(0, txt) # Fake if version[0] >= 3 or (version[0] == 2 and version[1] >= 6): diff --git a/python/helpers/pycharm_generator_utils/util_methods.py b/python/helpers/pycharm_generator_utils/util_methods.py index a7e0877ad017..ab93f31bec44 100644 --- a/python/helpers/pycharm_generator_utils/util_methods.py +++ b/python/helpers/pycharm_generator_utils/util_methods.py @@ -135,6 +135,39 @@ class __method(object): """ return txt + +def create_coroutine(): + if version[0] == 3 and version[1] >= 5: + return """ +class __coroutine(object): + '''A mock class representing coroutine type.''' + + def __init__(self): + self.__name__ = '' + self.__qualname__ = '' + self.cr_await = None + self.cr_frame = None + self.cr_running = False + self.cr_code = None + + def __await__(self): + return [] + + def __iter__(self): + return [] + + def close(self): + pass + + def send(self, value): + pass + + def throw(self, type, value=None, traceback=None): + pass +""" + return "" + + def _searchbases(cls, accum): # logic copied from inspect.py if cls not in accum: diff --git a/python/helpers/python-skeletons/__builtin__.py b/python/helpers/python-skeletons/__builtin__.py index 478e6c3f61c5..e42af9273673 100644 --- a/python/helpers/python-skeletons/__builtin__.py +++ b/python/helpers/python-skeletons/__builtin__.py @@ -382,7 +382,7 @@ class enumerate(object): :type iterable: collections.Iterable[T] :type start: numbers.Integral - :rtype: enumerate[int, T] + :rtype: enumerate[T] """ pass @@ -396,7 +396,7 @@ class enumerate(object): def __iter__(self): """x.__iter__() <==> iter(x). - :rtype: enumerate[int, T] + :rtype: collections.Iterator[(int, T)] """ return self @@ -2026,6 +2026,12 @@ class list(object): """ return [] + def __iter__(self): + """ + :rtype: collections.Iterator[T] + """ + return [] + def __getitem__(self, y): """y-th item of x, origin 0. @@ -2450,18 +2456,20 @@ class file(object): class __generator(object): """A mock class representing the generator function type.""" - def __init__(self, value): + def __init__(self): """Create a generator object. - :type value: T - :rtype: __generator[T] + :rtype: __generator[T, U, V] """ self.gi_code = None self.gi_frame = None self.gi_running = 0 def __iter__(self): - """Defined to support iteration over container.""" + """Defined to support iteration over container. + + :rtype: collections.Iterator[T] + """ pass def next(self): @@ -2483,7 +2491,8 @@ class __generator(object): """Resumes the generator and "sends" a value that becomes the result of the current yield-expression. - :rtype: T + :type value: U + :rtype: None """ pass @@ -2494,6 +2503,7 @@ class __generator(object): """ pass + class __function(object): """A mock class representing function type.""" diff --git a/python/helpers/python-skeletons/builtins.py b/python/helpers/python-skeletons/builtins.py index 1cfb32ca6a4b..ef1b30c35c23 100644 --- a/python/helpers/python-skeletons/builtins.py +++ b/python/helpers/python-skeletons/builtins.py @@ -370,7 +370,7 @@ class enumerate(object): :type iterable: collections.Iterable[T] :type start: int | long - :rtype: enumerate[int, T] + :rtype: enumerate[T] """ pass @@ -384,7 +384,7 @@ class enumerate(object): def __iter__(self): """x.__iter__() <==> iter(x). - :rtype: enumerate[int, T] + :rtype: collections.Iterator[(int, T)] """ return self @@ -1716,6 +1716,12 @@ class list(object): """ return [] + def __iter__(self): + """ + :rtype: collections.Iterator[T] + """ + return [] + def __getitem__(self, y): """y-th item of x, origin 0. @@ -1999,18 +2005,20 @@ class dict(object): class __generator(object): """A mock class representing the generator function type.""" - def __init__(self, value): + def __init__(self): """Create a generator object. - :type value: T - :rtype: __generator[T] + :rtype: __generator[T, U, V] """ self.gi_code = None self.gi_frame = None self.gi_running = 0 def __iter__(self): - """Defined to support iteration over container.""" + """Defined to support iteration over container. + + :rtype: collections.Iterator[T] + """ pass def __next__(self): @@ -2032,7 +2040,8 @@ class __generator(object): """Resumes the generator and "sends" a value that becomes the result of the current yield-expression. - :rtype: T + :type value: U + :rtype: None """ pass @@ -2043,6 +2052,7 @@ class __generator(object): """ pass + class __function(object): """A mock class representing function type.""" diff --git a/python/helpers/required_gen_version b/python/helpers/required_gen_version index 81c8c905f82f..b9dc50e70d5e 100644 --- a/python/helpers/required_gen_version +++ b/python/helpers/required_gen_version @@ -6,7 +6,7 @@ (default) 1.127 # anything not explicitly marked -(built-in) 1.130 # skeletons of all built-in modules are built together +(built-in) 1.137 # skeletons of all built-in modules are built together # Note: modules like itertools, etc are "(built-in)" and are ignored if given separately _fileio 1.127 diff --git a/python/testData/MockSdk3.4/python_stubs/builtins.py b/python/testData/MockSdk3.4/python_stubs/builtins.py index bc425c814cd8..7309cbaf3c66 100644 --- a/python/testData/MockSdk3.4/python_stubs/builtins.py +++ b/python/testData/MockSdk3.4/python_stubs/builtins.py @@ -1,7 +1,7 @@ # encoding: utf-8 # module builtins # from (built-in) -# by generator 1.135 +# by generator 1.137 """ Built-in functions, exceptions, and other objects. @@ -16,88 +16,73 @@ Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices. # definition of __debug__ omitted # functions -from object import object -def abs(number): # real signature unknown; restored from __doc__ - """ - abs(number) -> number - - Return the absolute value of the argument. - """ - return 0 +def abs(*args, **kwargs): # real signature unknown + """ Return the absolute value of the argument. """ + pass -def all(iterable): # real signature unknown; restored from __doc__ +def all(*args, **kwargs): # real signature unknown """ - all(iterable) -> bool - Return True if bool(x) is True for all values x in the iterable. + If the iterable is empty, return True. """ - return False + pass -def any(iterable): # real signature unknown; restored from __doc__ +def any(*args, **kwargs): # real signature unknown """ - any(iterable) -> bool - Return True if bool(x) is True for any x in the iterable. + If the iterable is empty, return False. """ - return False + pass -def ascii(p_object): # real signature unknown; restored from __doc__ +def ascii(*args, **kwargs): # real signature unknown """ - ascii(object) -> string + Return an ASCII-only representation of an object. As repr(), return a string containing a printable representation of an object, but escape the non-ASCII characters in the string returned by - repr() using \x, \u or \U escapes. This generates a string similar + repr() using \\x, \\u or \\U escapes. This generates a string similar to that returned by repr() in Python 2. """ - return "" + pass -def bin(number): # real signature unknown; restored from __doc__ +def bin(*args, **kwargs): # real signature unknown; NOTE: unreliably restored from __doc__ """ - bin(number) -> string - Return the binary representation of an integer. >>> bin(2796202) '0b1010101010101010101010' """ - return "" + pass -def callable(p_object): # real signature unknown; restored from __doc__ +def callable(i_e_, some_kind_of_function): # real signature unknown; restored from __doc__ """ - callable(object) -> bool - Return whether the object is callable (i.e., some kind of function). + Note that classes are callable, as are instances of classes with a __call__() method. """ - return False + pass -def chr(i): # real signature unknown; restored from __doc__ - """ - chr(i) -> Unicode character - - Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. - """ - return "" +def chr(*args, **kwargs): # real signature unknown + """ Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. """ + pass -def compile(source, filename, mode, flags=None, dont_inherit=None): # real signature unknown; restored from __doc__ +def compile(*args, **kwargs): # real signature unknown """ - compile(source, filename, mode[, flags[, dont_inherit]]) -> code object + Compile source into a code object that can be executed by exec() or eval(). - Compile the source (a Python module, statement or expression) - into a code object that can be executed by exec() or eval(). + The source code may represent a Python module, statement or expression. The filename will be used for run-time error messages. The mode must be 'exec' to compile a module, 'single' to compile a single (interactive) statement, or 'eval' to compile an expression. The flags argument, if present, controls which future statements influence the compilation of the code. - The dont_inherit argument, if non-zero, stops the compilation inheriting + The dont_inherit argument, if true, stops the compilation inheriting the effects of any future statements in effect in the code calling - compile; if absent or zero these statements do influence the compilation, + compile; if absent or false these statements do influence the compilation, in addition to any features explicitly specified. """ pass @@ -116,12 +101,11 @@ def credits(*args, **kwargs): # real signature unknown """ pass -def delattr(p_object, name): # real signature unknown; restored from __doc__ +def delattr(x, y): # real signature unknown; restored from __doc__ """ - delattr(object, name) + Deletes the named attribute from the given object. - Delete a named attribute on an object; delattr(x, 'y') is equivalent to - ``del x.y''. + delattr(x, 'y') is equivalent to ``del x.y'' """ pass @@ -143,18 +127,13 @@ def dir(p_object=None): # real signature unknown; restored from __doc__ return [] def divmod(x, y): # known case of builtins.divmod - """ - divmod(x, y) -> (div, mod) - - Return the tuple ((x-x%y)/y, x%y). Invariant: div*y + mod == x. - """ + """ Return the tuple ((x-x%y)/y, x%y). Invariant: div*y + mod == x. """ return (0, 0) -def eval(source, globals=None, locals=None): # real signature unknown; restored from __doc__ +def eval(*args, **kwargs): # real signature unknown """ - eval(source[, globals[, locals]]) -> value + Evaluate the given source in the context of globals and locals. - Evaluate the source in the context of globals and locals. The source may be a string representing a Python expression or a code object as returned by compile(). The globals must be a dictionary and locals can be any mapping, @@ -163,28 +142,28 @@ def eval(source, globals=None, locals=None): # real signature unknown; restored """ pass -def exec(p_object, globals=None, locals=None): # real signature unknown; restored from __doc__ +def exec(*args, **kwargs): # real signature unknown """ - exec(object[, globals[, locals]]) + Execute the given source in the context of globals and locals. - Read and execute code from an object, which can be a string or a code - object. - The globals and locals are dictionaries, defaulting to the current - globals and locals. If only globals is given, locals defaults to it. + The source may be a string representing one or more Python statements + or a code object as returned by compile(). + The globals must be a dictionary and locals can be any mapping, + defaulting to the current globals and locals. + If only globals is given, locals defaults to it. """ pass def exit(*args, **kwargs): # real signature unknown pass -def format(value, format_spec=None): # real signature unknown; restored from __doc__ +def format(*args, **kwargs): # real signature unknown """ - format(value[, format_spec]) -> string + Return value.__format__(format_spec) - Returns value.__format__(format_spec) - format_spec defaults to "" + format_spec defaults to the empty string """ - return "" + pass def getattr(object, name, default=None): # known special case of getattr """ @@ -196,31 +175,31 @@ def getattr(object, name, default=None): # known special case of getattr """ pass -def globals(): # real signature unknown; restored from __doc__ +def globals(*args, **kwargs): # real signature unknown """ - globals() -> dictionary - Return the dictionary containing the current scope's global variables. - """ - return {} - -def hasattr(p_object, name): # real signature unknown; restored from __doc__ - """ - hasattr(object, name) -> bool + NOTE: Updates to this dictionary *will* affect name lookups in the current + global scope and vice-versa. + """ + pass + +def hasattr(*args, **kwargs): # real signature unknown + """ Return whether the object has an attribute with the given name. - (This is done by calling getattr(object, name) and catching AttributeError.) - """ - return False - -def hash(p_object): # real signature unknown; restored from __doc__ - """ - hash(object) -> integer - Return a hash value for the object. Two objects with the same value have - the same hash value. The reverse is not necessarily true, but likely. + This is done by calling getattr(obj, name) and catching AttributeError. """ - return 0 + pass + +def hash(*args, **kwargs): # real signature unknown + """ + Return the hash value for the given object. + + Two objects that compare equal must also have the same hash value, but the + reverse is not necessarily true. + """ + pass def help(): # real signature unknown; restored from __doc__ """ @@ -234,57 +213,55 @@ def help(): # real signature unknown; restored from __doc__ """ pass -def hex(number): # real signature unknown; restored from __doc__ +def hex(*args, **kwargs): # real signature unknown; NOTE: unreliably restored from __doc__ """ - hex(number) -> string - Return the hexadecimal representation of an integer. - >>> hex(3735928559) - '0xdeadbeef' + >>> hex(12648430) + '0xc0ffee' """ - return "" + pass -def id(p_object): # real signature unknown; restored from __doc__ +def id(*args, **kwargs): # real signature unknown """ - id(object) -> integer + Return the identity of an object. - Return the identity of an object. This is guaranteed to be unique among - simultaneously existing objects. (Hint: it's the object's memory address.) + This is guaranteed to be unique among simultaneously existing objects. + (CPython uses the object's memory address.) """ - return 0 + pass -def input(prompt=None): # real signature unknown; restored from __doc__ +def input(*args, **kwargs): # real signature unknown """ - input([prompt]) -> string - Read a string from standard input. The trailing newline is stripped. - If the user hits EOF (Unix: Ctl-D, Windows: Ctl-Z+Return), raise EOFError. - On Unix, GNU readline is used if enabled. The prompt string, if given, - is printed without a trailing newline before reading. - """ - return "" - -def isinstance(p_object, class_or_type_or_tuple): # real signature unknown; restored from __doc__ - """ - isinstance(object, class-or-type-or-tuple) -> bool + The prompt string, if given, is printed to standard output without a + trailing newline before reading input. + + If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError. + On *nix systems, readline is used if available. + """ + pass + +def isinstance(x, A_tuple): # real signature unknown; restored from __doc__ + """ Return whether an object is an instance of a class or of a subclass thereof. - With a type as second argument, return whether that is the object's type. - The form using a tuple, isinstance(x, (A, B, ...)), is a shortcut for - isinstance(x, A) or isinstance(x, B) or ... (etc.). - """ - return False - -def issubclass(C, B): # real signature unknown; restored from __doc__ - """ - issubclass(C, B) -> bool - Return whether class C is a subclass (i.e., a derived class) of class B. - When using a tuple as the second argument issubclass(X, (A, B, ...)), - is a shortcut for issubclass(X, A) or issubclass(X, B) or ... (etc.). + A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to + check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B) + or ...`` etc. """ - return False + pass + +def issubclass(x, A_tuple): # real signature unknown; restored from __doc__ + """ + Return whether 'cls' is a derived from another class or is the same class. + + A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to + check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B) + or ...`` etc. + """ + pass def iter(source, sentinel=None): # known special case of iter """ @@ -297,12 +274,8 @@ def iter(source, sentinel=None): # known special case of iter """ pass -def len(p_object): # real signature unknown; restored from __doc__ - """ - len(object) - - Return the number of items of a sequence or mapping. - """ +def len(*args, **kwargs): # real signature unknown + """ Return the number of items in a container. """ pass def license(*args, **kwargs): # real signature unknown @@ -312,30 +285,36 @@ def license(*args, **kwargs): # real signature unknown """ pass -def locals(): # real signature unknown; restored from __doc__ +def locals(*args, **kwargs): # real signature unknown """ - locals() -> dictionary + Return a dictionary containing the current scope's local variables. - Update and return a dictionary containing the current scope's local variables. + NOTE: Whether or not updates to this dictionary will affect name lookups in + the local scope and vice-versa is *implementation dependent* and not + covered by any backwards compatibility guarantees. """ - return {} + pass def max(*args, key=None): # known special case of max """ - max(iterable[, key=func]) -> value - max(a, b, c, ...[, key=func]) -> value + max(iterable, *[, default=obj, key=func]) -> value + max(arg1, arg2, *args, *[, key=func]) -> value - With a single iterable argument, return its largest item. + With a single iterable argument, return its biggest item. The + default keyword-only argument specifies an object to return if + the provided iterable is empty. With two or more arguments, return the largest argument. """ pass def min(*args, key=None): # known special case of min """ - min(iterable[, key=func]) -> value - min(a, b, c, ...[, key=func]) -> value + min(iterable, *[, default=obj, key=func]) -> value + min(arg1, arg2, *args, *[, key=func]) -> value - With a single iterable argument, return its smallest item. + With a single iterable argument, return its smallest item. The + default keyword-only argument specifies an object to return if + the provided iterable is empty. With two or more arguments, return the smallest argument. """ pass @@ -349,22 +328,17 @@ def next(iterator, default=None): # real signature unknown; restored from __doc_ """ pass -def oct(number): # real signature unknown; restored from __doc__ +def oct(*args, **kwargs): # real signature unknown; NOTE: unreliably restored from __doc__ """ - oct(number) -> string - Return the octal representation of an integer. >>> oct(342391) '0o1234567' """ - return "" + pass def open(file, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True): # known special case of open """ - open(file, mode='r', buffering=-1, encoding=None, - errors=None, newline=None, closefd=True, opener=None) -> file object - Open file and return a stream. Raise IOError upon failure. file is either a text or byte string giving the name (and the path @@ -486,22 +460,18 @@ def open(file, mode='r', buffering=None, encoding=None, errors=None, newline=Non """ pass -def ord(c): # real signature unknown; restored from __doc__ - """ - ord(c) -> integer - - Return the integer ordinal of a one-character string. - """ - return 0 +def ord(*args, **kwargs): # real signature unknown + """ Return the Unicode code point for a one-character string. """ + pass -def pow(x, y, z=None): # real signature unknown; restored from __doc__ +def pow(*args, **kwargs): # real signature unknown """ - pow(x, y[, z]) -> number + Equivalent to x**y (with two arguments) or x**y % z (with three arguments) - With two arguments, equivalent to x**y. With three arguments, - equivalent to (x**y) % z, but may be more efficient (e.g. for ints). + Some types, such as ints, are able to use a more efficient algorithm when + invoked using the three argument form. """ - return 0 + pass def print(*args, sep=' ', end='\n', file=None): # known special case of print """ @@ -519,14 +489,13 @@ def print(*args, sep=' ', end='\n', file=None): # known special case of print def quit(*args, **kwargs): # real signature unknown pass -def repr(p_object): # real signature unknown; restored from __doc__ +def repr(obj): # real signature unknown; restored from __doc__ """ - repr(object) -> string - Return the canonical string representation of the object. - For most object types, eval(repr(object)) == object. + + For many object types, including most builtins, eval(repr(obj)) == obj. """ - return "" + pass def round(number, ndigits=None): # real signature unknown; restored from __doc__ """ @@ -538,26 +507,30 @@ def round(number, ndigits=None): # real signature unknown; restored from __doc__ """ return 0 -def setattr(p_object, name, value): # real signature unknown; restored from __doc__ +def setattr(x, y, v): # real signature unknown; restored from __doc__ """ - setattr(object, name, value) + Sets the named attribute on the given object to the specified value. - Set a named attribute on an object; setattr(x, 'y', v) is equivalent to - ``x.y = v''. + setattr(x, 'y', v) is equivalent to ``x.y = v'' """ pass -def sorted(iterable, key=None, reverse=False): # real signature unknown; restored from __doc__ - """ sorted(iterable, key=None, reverse=False) --> new sorted list """ +def sorted(*args, **kwargs): # real signature unknown + """ + Return a new list containing all items from the iterable in ascending order. + + A custom key function can be supplied to customise the sort order, and the + reverse flag can be set to request the result in descending order. + """ pass -def sum(iterable, start=None): # real signature unknown; restored from __doc__ +def sum(*args, **kwargs): # real signature unknown """ - sum(iterable[, start]) -> value + Return the sum of a 'start' value (default: 0) plus an iterable of numbers - Return the sum of an iterable of numbers (NOT strings) plus the value - of parameter 'start' (which defaults to 0). When the iterable is - empty, return start. + When the iterable is empty, return the start value. + This function is intended specifically for use with numeric values and may + reject non-numeric types. """ pass @@ -659,6 +632,33 @@ class __method(object): self.__self__ = None +class __coroutine(object): + '''A mock class representing coroutine type.''' + + def __init__(self): + self.__name__ = '' + self.__qualname__ = '' + self.cr_await = None + self.cr_frame = None + self.cr_running = False + self.cr_code = None + + def __await__(self): + return [] + + def __iter__(self): + return [] + + def close(self): + pass + + def send(self, value): + pass + + def throw(self, type, value=None, traceback=None): + pass + + class __namedtuple(tuple): '''A mock base class for named tuples.''' @@ -791,8 +791,6 @@ class object: __module__ = '' -from .object import object - class BaseException(object): """ Common base class for all exceptions """ def with_traceback(self, tb): # real signature unknown; restored from __doc__ @@ -852,8 +850,6 @@ class BaseException(object): __dict__ = None # (!) real value is '' -from .BaseException import BaseException - class Exception(BaseException): """ Common base class for all non-exit exceptions. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -865,8 +861,6 @@ class Exception(BaseException): pass -from .Exception import Exception - class ArithmeticError(Exception): """ Base class for arithmetic errors. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -878,8 +872,6 @@ class ArithmeticError(Exception): pass -from .Exception import Exception - class AssertionError(Exception): """ Assertion failed. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -891,8 +883,6 @@ class AssertionError(Exception): pass -from .Exception import Exception - class AttributeError(Exception): """ Attribute not found. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -904,9 +894,7 @@ class AttributeError(Exception): pass -from .Exception import Exception - -class WindowsError(Exception): +class OSError(Exception): """ Base class for I/O related errors. """ def __init__(self, *args, **kwargs): # real signature unknown pass @@ -937,30 +925,20 @@ class WindowsError(Exception): strerror = property(lambda self: object(), lambda self, v: None, lambda self: None) # default """exception strerror""" - winerror = property(lambda self: object(), lambda self, v: None, lambda self: None) # default - """Win32 exception code""" - -OSError = WindowsError +IOError = OSError -IOError = WindowsError +EnvironmentError = OSError -EnvironmentError = WindowsError - - -from .OSError import OSError - class BlockingIOError(OSError): """ I/O operation would block. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .object import object - class int(object): """ int(x=0) -> integer @@ -1001,9 +979,7 @@ class int(object): Return the integer represented by the given array of bytes. - The bytes argument must either support the buffer protocol or be an - iterable object producing bytes. Bytes and bytearray are examples of - built-in objects that support the buffer protocol. + The bytes argument must be a bytes-like object (e.g. bytes or bytearray). The byteorder argument determines the byte order used to represent the integer. If byteorder is 'big', the most significant byte is at the @@ -1282,8 +1258,6 @@ class int(object): -from .int import int - class bool(int): """ bool(x) -> bool @@ -1333,24 +1307,18 @@ class bool(int): pass -from .OSError import OSError - class ConnectionError(OSError): """ Connection error. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .ConnectionError import ConnectionError - class BrokenPipeError(ConnectionError): """ Broken pipe. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .Exception import Exception - class BufferError(Exception): """ Buffer error. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -1362,8 +1330,6 @@ class BufferError(Exception): pass -from .object import object - class bytearray(object): """ bytearray(iterable_of_ints) -> bytearray @@ -1379,11 +1345,12 @@ class bytearray(object): - any object implementing the buffer API. - an integer """ - def append(self, p_int): # real signature unknown; restored from __doc__ + def append(self, *args, **kwargs): # real signature unknown """ - B.append(int) -> None + Append a single item to the end of the bytearray. - Append a single item to the end of B. + item + The item to be appended. """ pass @@ -1405,21 +1372,13 @@ class bytearray(object): """ pass - def clear(self): # real signature unknown; restored from __doc__ - """ - B.clear() -> None - - Remove all items from B. - """ + def clear(self, *args, **kwargs): # real signature unknown + """ Remove all items from the bytearray. """ pass - def copy(self): # real signature unknown; restored from __doc__ - """ - B.copy() -> bytearray - - Return a copy of B. - """ - return bytearray + def copy(self, *args, **kwargs): # real signature unknown + """ Return a copy of B. """ + pass def count(self, sub, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -1431,18 +1390,20 @@ class bytearray(object): """ return 0 - def decode(self, encoding='utf-8', errors='strict'): # real signature unknown; restored from __doc__ + def decode(self, *args, **kwargs): # real signature unknown """ - B.decode(encoding='utf-8', errors='strict') -> str + Decode the bytearray using the codec registered for encoding. - Decode B using the codec registered for encoding. Default encoding - is 'utf-8'. errors may be given to set a different error - handling scheme. Default is 'strict' meaning that encoding errors raise - a UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that is - able to handle UnicodeDecodeErrors. + encoding + The encoding with which to decode the bytearray. + errors + The error handling scheme to use for the handling of decoding errors. + The default is 'strict' meaning that decoding errors raise a + UnicodeDecodeError. Other possible values are 'ignore' and 'replace' + as well as any other name registered with codecs.register_error that + can handle UnicodeDecodeErrors. """ - return "" + pass def endswith(self, suffix, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -1464,12 +1425,12 @@ class bytearray(object): """ pass - def extend(self, iterable_of_ints): # real signature unknown; restored from __doc__ + def extend(self, *args, **kwargs): # real signature unknown """ - B.extend(iterable_of_ints) -> None + Append all the items from the iterator or sequence to the end of the bytearray. - Append all the elements from the iterator or sequence to the - end of B. + iterable_of_ints + The iterable of items to append. """ pass @@ -1486,15 +1447,23 @@ class bytearray(object): return 0 @classmethod # known case - def fromhex(cls, string): # real signature unknown; restored from __doc__ + def fromhex(cls, *args, **kwargs): # real signature unknown; NOTE: unreliably restored from __doc__ """ - bytearray.fromhex(string) -> bytearray (static method) - Create a bytearray object from a string of hexadecimal numbers. + Spaces between two numbers are accepted. - Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\xb9\x01\xef'). + Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') """ - return bytearray + pass + + def hex(self): # real signature unknown; restored from __doc__ + """ + B.hex() -> string + + Create a string of hexadecimal numbers from a bytearray object. + Example: bytearray([0xb9, 0x01, 0xef]).hex() -> 'b901ef'. + """ + return "" def index(self, sub, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -1504,11 +1473,14 @@ class bytearray(object): """ return 0 - def insert(self, index, p_int): # real signature unknown; restored from __doc__ + def insert(self, *args, **kwargs): # real signature unknown """ - B.insert(index, int) -> None - Insert a single item into the bytearray before the given index. + + index + The index where the value is to be inserted. + item + The item to be inserted. """ pass @@ -1577,14 +1549,15 @@ class bytearray(object): """ return False - def join(self, iterable_of_bytes): # real signature unknown; restored from __doc__ + def join(self, *args, **kwargs): # real signature unknown """ - B.join(iterable_of_bytes) -> bytearray + Concatenate any number of bytes/bytearray objects. - Concatenate any number of bytes/bytearray objects, with B - in between each pair, and return the result as a new bytearray. + The bytearray whose method is called is inserted in between each pair. + + The result is returned as a new bytearray object. """ - return bytearray + pass def ljust(self, width, fillchar=None): # real signature unknown; restored from __doc__ """ @@ -1603,73 +1576,77 @@ class bytearray(object): """ pass - def lstrip(self, bytes=None): # real signature unknown; restored from __doc__ + def lstrip(self, *args, **kwargs): # real signature unknown """ - B.lstrip([bytes]) -> bytearray + Strip leading bytes contained in the argument. - Strip leading bytes contained in the argument - and return the result as a new bytearray. - If the argument is omitted, strip leading ASCII whitespace. + If the argument is omitted or None, strip leading ASCII whitespace. """ - return bytearray + pass @staticmethod # known case - def maketrans(frm, to): # real signature unknown; restored from __doc__ + def maketrans(*args, **kwargs): # real signature unknown """ - B.maketrans(frm, to) -> translation table + Return a translation table useable for the bytes or bytearray translate method. + + The returned table will be one where each byte in frm is mapped to the byte at + the same position in to. - Return a translation table (a bytes object of length 256) suitable - for use in the bytes or bytearray translate method where each byte - in frm is mapped to the byte at the same position in to. The bytes objects frm and to must be of the same length. """ pass - def partition(self, sep): # real signature unknown; restored from __doc__ + def partition(self, *args, **kwargs): # real signature unknown """ - B.partition(sep) -> (head, sep, tail) + Partition the bytearray into three parts using the given separator. - Search for the separator sep in B, and return the part before it, - the separator itself, and the part after it. If the separator is not - found, returns B and two empty bytearray objects. + This will search for the separator sep in the bytearray. If the separator is + found, returns a 3-tuple containing the part before the separator, the + separator itself, and the part after it. + + If the separator is not found, returns a 3-tuple containing the original + bytearray object and two empty bytearray objects. """ pass - def pop(self, index=None): # real signature unknown; restored from __doc__ + def pop(self, *args, **kwargs): # real signature unknown """ - B.pop([index]) -> int + Remove and return a single item from B. - Remove and return a single item from B. If no index - argument is given, will pop the last value. - """ - return 0 - - def remove(self, p_int): # real signature unknown; restored from __doc__ - """ - B.remove(int) -> None + index + The index from where to remove the item. + -1 (the default value) means remove the last item. - Remove the first occurrence of a value in B. + If no index argument is given, will pop the last item. """ pass - def replace(self, old, new, count=None): # real signature unknown; restored from __doc__ + def remove(self, *args, **kwargs): # real signature unknown """ - B.replace(old, new[, count]) -> bytearray + Remove the first occurrence of a value in the bytearray. - Return a copy of B with all occurrences of subsection - old replaced by new. If the optional argument count is - given, only the first count occurrences are replaced. + value + The value to remove. """ - return bytearray + pass - def reverse(self): # real signature unknown; restored from __doc__ + def replace(self, *args, **kwargs): # real signature unknown """ - B.reverse() -> None + Return a copy with all occurrences of substring old replaced by new. - Reverse the order of the values in B in place. + count + Maximum number of occurrences to replace. + -1 (the default value) means replace all occurrences. + + If the optional argument count is given, only the first count occurrences are + replaced. """ pass + def reverse(self, *args, **kwargs): # real signature unknown + """ Reverse the order of the values in B in place. """ + pass + def rfind(self, sub, start=None, end=None): # real signature unknown; restored from __doc__ """ B.rfind(sub[, start[, end]]) -> int @@ -1699,59 +1676,65 @@ class bytearray(object): """ pass - def rpartition(self, sep): # real signature unknown; restored from __doc__ + def rpartition(self, *args, **kwargs): # real signature unknown """ - B.rpartition(sep) -> (head, sep, tail) + Partition the bytes into three parts using the given separator. - Search for the separator sep in B, starting at the end of B, - and return the part before it, the separator itself, and the - part after it. If the separator is not found, returns two empty - bytearray objects and B. + This will search for the separator sep in the bytearray, starting and the end. + If the separator is found, returns a 3-tuple containing the part before the + separator, the separator itself, and the part after it. + + If the separator is not found, returns a 3-tuple containing two empty bytearray + objects and the original bytearray object. """ pass - def rsplit(self, sep=None, maxsplit=-1): # real signature unknown; restored from __doc__ + def rsplit(self, *args, **kwargs): # real signature unknown """ - B.rsplit(sep=None, maxsplit=-1) -> list of bytearrays + Return a list of the sections in the bytearray, using sep as the delimiter. - Return a list of the sections in B, using sep as the delimiter, - starting at the end of B and working to the front. - If sep is not given, B is split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). - If maxsplit is given, at most maxsplit splits are done. + sep + The delimiter according which to split the bytearray. + None (the default value) means split on ASCII whitespace characters + (space, tab, return, newline, formfeed, vertical tab). + maxsplit + Maximum number of splits to do. + -1 (the default value) means no limit. + + Splitting is done starting at the end of the bytearray and working to the front. """ - return [] + pass - def rstrip(self, bytes=None): # real signature unknown; restored from __doc__ + def rstrip(self, *args, **kwargs): # real signature unknown """ - B.rstrip([bytes]) -> bytearray + Strip trailing bytes contained in the argument. - Strip trailing bytes contained in the argument - and return the result as a new bytearray. - If the argument is omitted, strip trailing ASCII whitespace. + If the argument is omitted or None, strip trailing ASCII whitespace. """ - return bytearray + pass - def split(self, sep=None, maxsplit=-1): # real signature unknown; restored from __doc__ + def split(self, *args, **kwargs): # real signature unknown """ - B.split(sep=None, maxsplit=-1) -> list of bytearrays + Return a list of the sections in the bytearray, using sep as the delimiter. - Return a list of the sections in B, using sep as the delimiter. - If sep is not given, B is split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). - If maxsplit is given, at most maxsplit splits are done. + sep + The delimiter according which to split the bytearray. + None (the default value) means split on ASCII whitespace characters + (space, tab, return, newline, formfeed, vertical tab). + maxsplit + Maximum number of splits to do. + -1 (the default value) means no limit. """ - return [] + pass - def splitlines(self, keepends=None): # real signature unknown; restored from __doc__ + def splitlines(self, *args, **kwargs): # real signature unknown """ - B.splitlines([keepends]) -> list of lines + Return a list of the lines in the bytearray, breaking at line boundaries. - Return a list of the lines in B, breaking at line boundaries. - Line breaks are not included in the resulting list unless keepends - is given and true. + Line breaks are not included in the resulting list unless keepends is given and + true. """ - return [] + pass def startswith(self, prefix, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -1764,15 +1747,13 @@ class bytearray(object): """ return False - def strip(self, bytes=None): # real signature unknown; restored from __doc__ + def strip(self, *args, **kwargs): # real signature unknown """ - B.strip([bytes]) -> bytearray + Strip leading and trailing bytes contained in the argument. - Strip leading and trailing bytes contained in the argument - and return the result as a new bytearray. - If the argument is omitted, strip ASCII whitespace. + If the argument is omitted or None, strip leading and trailing ASCII whitespace. """ - return bytearray + pass def swapcase(self): # real signature unknown; restored from __doc__ """ @@ -1794,14 +1775,16 @@ class bytearray(object): def translate(self, table, deletechars=None): # real signature unknown; restored from __doc__ """ - B.translate(table[, deletechars]) -> bytearray + translate(table, [deletechars]) + Return a copy with each character mapped by the given translation table. - Return a copy of B, where all characters occurring in the - optional argument deletechars are removed, and the remaining - characters have been mapped through the given translation - table, which must be a bytes object of length 256. + table + Translation table, which must be a bytes object of length 256. + + All characters occurring in the optional argument deletechars are removed. + The remaining characters are mapped through the given translation table. """ - return bytearray + pass def upper(self): # real signature unknown; restored from __doc__ """ @@ -1902,6 +1885,10 @@ class bytearray(object): """ Return self int - - Returns the size of B in memory, in bytes - """ - return 0 + def __sizeof__(self, *args, **kwargs): # real signature unknown + """ Returns the size of the bytearray object in memory, in bytes. """ + pass def __str__(self, *args, **kwargs): # real signature unknown """ Return str(self). """ @@ -1950,8 +1937,6 @@ class bytearray(object): __hash__ = None -from .object import object - class bytes(object): """ bytes(iterable_of_ints) -> bytes @@ -1994,18 +1979,20 @@ class bytes(object): """ return 0 - def decode(self, encoding='utf-8', errors='strict'): # real signature unknown; restored from __doc__ + def decode(self, *args, **kwargs): # real signature unknown """ - B.decode(encoding='utf-8', errors='strict') -> str + Decode the bytes using the codec registered for encoding. - Decode B using the codec registered for encoding. Default encoding - is 'utf-8'. errors may be given to set a different error - handling scheme. Default is 'strict' meaning that encoding errors raise - a UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registerd with codecs.register_error that is - able to handle UnicodeDecodeErrors. + encoding + The encoding with which to decode the bytes. + errors + The error handling scheme to use for the handling of decoding errors. + The default is 'strict' meaning that decoding errors raise a + UnicodeDecodeError. Other possible values are 'ignore' and 'replace' + as well as any other name registered with codecs.register_error that + can handle UnicodeDecodeErrors. """ - return "" + pass def endswith(self, suffix, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -2040,15 +2027,23 @@ class bytes(object): return 0 @classmethod # known case - def fromhex(cls, string): # real signature unknown; restored from __doc__ + def fromhex(cls, *args, **kwargs): # real signature unknown; NOTE: unreliably restored from __doc__ """ - bytes.fromhex(string) -> bytes - Create a bytes object from a string of hexadecimal numbers. + Spaces between two numbers are accepted. - Example: bytes.fromhex('B9 01EF') -> b'\xb9\x01\xef'. + Example: bytes.fromhex('B9 01EF') -> b'\\xb9\\x01\\xef'. """ - return b"" + pass + + def hex(self): # real signature unknown; restored from __doc__ + """ + B.hex() -> string + + Create a string of hexadecimal numbers from a bytes object. + Example: b'\xb9\x01\xef'.hex() -> 'b901ef'. + """ + return "" def index(self, sub, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -2123,14 +2118,17 @@ class bytes(object): """ return False - def join(self, iterable_of_bytes): # real signature unknown; restored from __doc__ + def join(self, *args, **kwargs): # real signature unknown; NOTE: unreliably restored from __doc__ """ - B.join(iterable_of_bytes) -> bytes + Concatenate any number of bytes objects. + + The bytes whose method is called is inserted in between each pair. + + The result is returned as a new bytes object. - Concatenate any number of bytes objects, with B in between each pair. Example: b'.'.join([b'ab', b'pq', b'rs']) -> b'ab.pq.rs'. """ - return b"" + pass def ljust(self, width, fillchar=None): # real signature unknown; restored from __doc__ """ @@ -2149,46 +2147,51 @@ class bytes(object): """ pass - def lstrip(self, bytes=None): # real signature unknown; restored from __doc__ + def lstrip(self, *args, **kwargs): # real signature unknown """ - B.lstrip([bytes]) -> bytes - Strip leading bytes contained in the argument. - If the argument is omitted, strip leading ASCII whitespace. + + If the argument is omitted or None, strip leading ASCII whitespace. """ - return b"" + pass @staticmethod # known case - def maketrans(frm, to): # real signature unknown; restored from __doc__ + def maketrans(*args, **kwargs): # real signature unknown """ - B.maketrans(frm, to) -> translation table + Return a translation table useable for the bytes or bytearray translate method. + + The returned table will be one where each byte in frm is mapped to the byte at + the same position in to. - Return a translation table (a bytes object of length 256) suitable - for use in the bytes or bytearray translate method where each byte - in frm is mapped to the byte at the same position in to. The bytes objects frm and to must be of the same length. """ pass - def partition(self, sep): # real signature unknown; restored from __doc__ + def partition(self, *args, **kwargs): # real signature unknown """ - B.partition(sep) -> (head, sep, tail) + Partition the bytes into three parts using the given separator. - Search for the separator sep in B, and return the part before it, - the separator itself, and the part after it. If the separator is not - found, returns B and two empty bytes objects. + This will search for the separator sep in the bytes. If the separator is found, + returns a 3-tuple containing the part before the separator, the separator + itself, and the part after it. + + If the separator is not found, returns a 3-tuple containing the original bytes + object and two empty bytes objects. """ pass - def replace(self, old, new, count=None): # real signature unknown; restored from __doc__ + def replace(self, *args, **kwargs): # real signature unknown """ - B.replace(old, new[, count]) -> bytes + Return a copy with all occurrences of substring old replaced by new. - Return a copy of B with all occurrences of subsection - old replaced by new. If the optional argument count is - given, only first count occurances are replaced. + count + Maximum number of occurrences to replace. + -1 (the default value) means replace all occurrences. + + If the optional argument count is given, only the first count occurrences are + replaced. """ - return b"" + pass def rfind(self, sub, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -2219,58 +2222,65 @@ class bytes(object): """ pass - def rpartition(self, sep): # real signature unknown; restored from __doc__ + def rpartition(self, *args, **kwargs): # real signature unknown """ - B.rpartition(sep) -> (head, sep, tail) + Partition the bytes into three parts using the given separator. - Search for the separator sep in B, starting at the end of B, - and return the part before it, the separator itself, and the - part after it. If the separator is not found, returns two empty - bytes objects and B. + This will search for the separator sep in the bytes, starting and the end. If + the separator is found, returns a 3-tuple containing the part before the + separator, the separator itself, and the part after it. + + If the separator is not found, returns a 3-tuple containing two empty bytes + objects and the original bytes object. """ pass - def rsplit(self, sep=None, maxsplit=-1): # real signature unknown; restored from __doc__ + def rsplit(self, *args, **kwargs): # real signature unknown """ - B.rsplit(sep=None, maxsplit=-1) -> list of bytes + Return a list of the sections in the bytes, using sep as the delimiter. - Return a list of the sections in B, using sep as the delimiter, - starting at the end of B and working to the front. - If sep is not given, B is split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). - If maxsplit is given, at most maxsplit splits are done. + sep + The delimiter according which to split the bytes. + None (the default value) means split on ASCII whitespace characters + (space, tab, return, newline, formfeed, vertical tab). + maxsplit + Maximum number of splits to do. + -1 (the default value) means no limit. + + Splitting is done starting at the end of the bytes and working to the front. """ - return [] + pass - def rstrip(self, bytes=None): # real signature unknown; restored from __doc__ + def rstrip(self, *args, **kwargs): # real signature unknown """ - B.rstrip([bytes]) -> bytes - Strip trailing bytes contained in the argument. - If the argument is omitted, strip trailing ASCII whitespace. - """ - return b"" - - def split(self, sep=None, maxsplit=-1): # real signature unknown; restored from __doc__ - """ - B.split(sep=None, maxsplit=-1) -> list of bytes - Return a list of the sections in B, using sep as the delimiter. - If sep is not specified or is None, B is split on ASCII whitespace - characters (space, tab, return, newline, formfeed, vertical tab). - If maxsplit is given, at most maxsplit splits are done. + If the argument is omitted or None, strip trailing ASCII whitespace. """ - return [] + pass - def splitlines(self, keepends=None): # real signature unknown; restored from __doc__ + def split(self, *args, **kwargs): # real signature unknown """ - B.splitlines([keepends]) -> list of lines + Return a list of the sections in the bytes, using sep as the delimiter. - Return a list of the lines in B, breaking at line boundaries. - Line breaks are not included in the resulting list unless keepends - is given and true. + sep + The delimiter according which to split the bytes. + None (the default value) means split on ASCII whitespace characters + (space, tab, return, newline, formfeed, vertical tab). + maxsplit + Maximum number of splits to do. + -1 (the default value) means no limit. """ - return [] + pass + + def splitlines(self, *args, **kwargs): # real signature unknown + """ + Return a list of the lines in the bytes, breaking at line boundaries. + + Line breaks are not included in the resulting list unless keepends is given and + true. + """ + pass def startswith(self, prefix, start=None, end=None): # real signature unknown; restored from __doc__ """ @@ -2283,14 +2293,13 @@ class bytes(object): """ return False - def strip(self, bytes=None): # real signature unknown; restored from __doc__ + def strip(self, *args, **kwargs): # real signature unknown """ - B.strip([bytes]) -> bytes - Strip leading and trailing bytes contained in the argument. - If the argument is omitted, strip leading and trailing ASCII whitespace. + + If the argument is omitted or None, strip leading and trailing ASCII whitespace. """ - return b"" + pass def swapcase(self): # real signature unknown; restored from __doc__ """ @@ -2312,14 +2321,16 @@ class bytes(object): def translate(self, table, deletechars=None): # real signature unknown; restored from __doc__ """ - B.translate(table[, deletechars]) -> bytes + translate(table, [deletechars]) + Return a copy with each character mapped by the given translation table. - Return a copy of B, where all characters occurring in the - optional argument deletechars are removed, and the remaining - characters have been mapped through the given translation - table, which must be a bytes object of length 256. + table + Translation table, which must be a bytes object of length 256. + + All characters occurring in the optional argument deletechars are removed. + The remaining characters are mapped through the given translation table. """ - return b"" + pass def upper(self): # real signature unknown; restored from __doc__ """ @@ -2406,6 +2417,10 @@ class bytes(object): """ Return self size of B in memory, in bytes """ + def __rmul__(self, *args, **kwargs): # real signature unknown + """ Return self*value. """ pass def __str__(self, *args, **kwargs): # real signature unknown @@ -2436,8 +2451,6 @@ class bytes(object): pass -from .Exception import Exception - class Warning(Exception): """ Base class for warning categories. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -2449,8 +2462,6 @@ class Warning(Exception): pass -from .Warning import Warning - class BytesWarning(Warning): """ Base class for warnings about bytes and buffer related problems, mostly @@ -2465,16 +2476,12 @@ class BytesWarning(Warning): pass -from .OSError import OSError - class ChildProcessError(OSError): """ Child process error. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .object import object - class classmethod(object): """ classmethod(function) -> method @@ -2517,8 +2524,6 @@ class classmethod(object): __dict__ = None # (!) real value is '' -from .object import object - class complex(object): """ complex(real[, imag]) -> complex number @@ -2695,32 +2700,24 @@ class complex(object): -from .ConnectionError import ConnectionError - class ConnectionAbortedError(ConnectionError): """ Connection aborted. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .ConnectionError import ConnectionError - class ConnectionRefusedError(ConnectionError): """ Connection refused. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .ConnectionError import ConnectionError - class ConnectionResetError(ConnectionError): """ Connection reset. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .Warning import Warning - class DeprecationWarning(Warning): """ Base class for warnings about deprecated features. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -2732,8 +2729,6 @@ class DeprecationWarning(Warning): pass -from .object import object - class dict(object): """ dict() -> new empty dictionary @@ -2885,8 +2880,6 @@ class dict(object): __hash__ = None -from .object import object - class enumerate(object): """ enumerate(iterable[, start]) -> iterator for index, value of iterable @@ -2923,8 +2916,6 @@ class enumerate(object): pass -from .Exception import Exception - class EOFError(Exception): """ Read beyond end of file. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -2936,24 +2927,18 @@ class EOFError(Exception): pass -from .OSError import OSError - class FileExistsError(OSError): """ File already exists. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .OSError import OSError - class FileNotFoundError(OSError): """ File not found. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .object import object - class filter(object): """ filter(function or None, iterable) --> filter object @@ -2986,8 +2971,6 @@ class filter(object): pass -from .object import object - class float(object): """ float(x) -> floating point number @@ -3238,8 +3221,6 @@ class float(object): -from .ArithmeticError import ArithmeticError - class FloatingPointError(ArithmeticError): """ Floating point operation failed. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3251,8 +3232,6 @@ class FloatingPointError(ArithmeticError): pass -from .object import object - class frozenset(object): """ frozenset() -> empty frozenset object @@ -3406,8 +3385,6 @@ class frozenset(object): pass -from .Warning import Warning - class FutureWarning(Warning): """ Base class for warnings about constructs that will change semantically @@ -3422,8 +3399,6 @@ class FutureWarning(Warning): pass -from .BaseException import BaseException - class GeneratorExit(BaseException): """ Request that a generator exit. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3435,8 +3410,6 @@ class GeneratorExit(BaseException): pass -from .Exception import Exception - class ImportError(Exception): """ Import can't find module, or can't find name in module. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3457,8 +3430,6 @@ class ImportError(Exception): -from .Warning import Warning - class ImportWarning(Warning): """ Base class for warnings about probable mistakes in module imports """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3470,8 +3441,6 @@ class ImportWarning(Warning): pass -from .Exception import Exception - class SyntaxError(Exception): """ Invalid syntax. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3501,16 +3470,12 @@ class SyntaxError(Exception): -from .SyntaxError import SyntaxError - class IndentationError(SyntaxError): """ Improper indentation. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .Exception import Exception - class LookupError(Exception): """ Base class for lookup errors. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3522,8 +3487,6 @@ class LookupError(Exception): pass -from .LookupError import LookupError - class IndexError(LookupError): """ Sequence index out of range. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3535,24 +3498,18 @@ class IndexError(LookupError): pass -from .OSError import OSError - class InterruptedError(OSError): """ Interrupted by signal. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .OSError import OSError - class IsADirectoryError(OSError): """ Operation doesn't work on directories. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .BaseException import BaseException - class KeyboardInterrupt(BaseException): """ Program interrupted by user. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3564,8 +3521,6 @@ class KeyboardInterrupt(BaseException): pass -from .LookupError import LookupError - class KeyError(LookupError): """ Mapping key not found. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3576,8 +3531,6 @@ class KeyError(LookupError): pass -from .object import object - class list(object): """ list() -> new empty list @@ -3736,8 +3689,6 @@ class list(object): __hash__ = None -from .object import object - class map(object): """ map(func, *iterables) --> map object @@ -3770,8 +3721,6 @@ class map(object): pass -from .Exception import Exception - class MemoryError(Exception): """ Out of memory. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3783,45 +3732,27 @@ class MemoryError(Exception): pass -from .object import object - class memoryview(object): - """ - memoryview(object) - - Create a new memoryview object which references the given object. - """ - def cast(self, format, shape=None): # real signature unknown; restored from __doc__ - """ - M.cast(format[, shape]) -> memoryview - - Cast a memoryview to a new format or shape. - """ - return memoryview - - def release(self): # real signature unknown; restored from __doc__ - """ - M.release() -> None - - Release the underlying buffer exposed by the memoryview object. - """ + """ Create a new memoryview object which references the given object. """ + def cast(self, *args, **kwargs): # real signature unknown + """ Cast a memoryview to a new format or shape. """ pass - def tobytes(self): # real signature unknown; restored from __doc__ - """ - M.tobytes() -> bytes - - Return the data in the buffer as a byte string. - """ - return b"" + def hex(self, *args, **kwargs): # real signature unknown + """ Return the data in the buffer as a string of hexadecimal numbers. """ + pass - def tolist(self): # real signature unknown; restored from __doc__ - """ - M.tolist() -> list - - Return the data in the buffer as a list of elements. - """ - return [] + def release(self, *args, **kwargs): # real signature unknown + """ Release the underlying buffer exposed by the memoryview object. """ + pass + + def tobytes(self, *args, **kwargs): # real signature unknown + """ Return the data in the buffer as a byte string. """ + pass + + def tolist(self, *args, **kwargs): # real signature unknown + """ Return the data in the buffer as a list of elements. """ + pass def __delitem__(self, *args, **kwargs): # real signature unknown """ Delete self[key]. """ @@ -3857,7 +3788,7 @@ class memoryview(object): """ Return hash(self). """ pass - def __init__(self, p_object): # real signature unknown; restored from __doc__ + def __init__(self, *args, **kwargs): # real signature unknown pass def __len__(self, *args, **kwargs): # real signature unknown @@ -3932,8 +3863,6 @@ class memoryview(object): -from .Exception import Exception - class NameError(Exception): """ Name not found globally. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3945,16 +3874,12 @@ class NameError(Exception): pass -from .OSError import OSError - class NotADirectoryError(OSError): """ Operation only works on directories. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .Exception import Exception - class RuntimeError(Exception): """ Unspecified run-time error. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3966,8 +3891,6 @@ class RuntimeError(Exception): pass -from .RuntimeError import RuntimeError - class NotImplementedError(RuntimeError): """ Method or function hasn't been implemented yet. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3979,8 +3902,6 @@ class NotImplementedError(RuntimeError): pass -from .ArithmeticError import ArithmeticError - class OverflowError(ArithmeticError): """ Result too large to be represented. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -3992,8 +3913,6 @@ class OverflowError(ArithmeticError): pass -from .Warning import Warning - class PendingDeprecationWarning(Warning): """ Base class for warnings about features which will be deprecated @@ -4008,24 +3927,18 @@ class PendingDeprecationWarning(Warning): pass -from .OSError import OSError - class PermissionError(OSError): """ Not enough permissions. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .OSError import OSError - class ProcessLookupError(OSError): """ Process not found. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .object import object - class property(object): """ property(fget=None, fset=None, fdel=None, doc=None) -> property attribute @@ -4129,14 +4042,16 @@ class property(object): -from .object import object - class range(object): """ range(stop) -> range object range(start, stop[, step]) -> range object - Return a virtual sequence of numbers from start to stop by step. + Return an object that produces a sequence of integers from start (inclusive) + to stop (exclusive) by step. range(i, j) produces i, i+1, i+2, ..., j-1. + start defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3. + These are exactly the valid indices for a list of 4 elements. + When step is given, it specifies the increment (or decrement). """ def count(self, value): # real signature unknown; restored from __doc__ """ rangeobject.count(value) -> integer -- return number of occurrences of value """ @@ -4224,7 +4139,16 @@ class range(object): -from .Exception import Exception +class RecursionError(RuntimeError): + """ Recursion limit exceeded. """ + def __init__(self, *args, **kwargs): # real signature unknown + pass + + @staticmethod # known case of __new__ + def __new__(*args, **kwargs): # real signature unknown + """ Create and return a new object. See help(type) for accurate signature. """ + pass + class ReferenceError(Exception): """ Weak ref proxy used after referent went away. """ @@ -4237,8 +4161,6 @@ class ReferenceError(Exception): pass -from .Warning import Warning - class ResourceWarning(Warning): """ Base class for warnings about resource usage. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -4250,8 +4172,6 @@ class ResourceWarning(Warning): pass -from .object import object - class reversed(object): """ reversed(sequence) -> reverse iterator over values of the sequence @@ -4291,8 +4211,6 @@ class reversed(object): pass -from .Warning import Warning - class RuntimeWarning(Warning): """ Base class for warnings about dubious runtime behavior. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -4304,8 +4222,6 @@ class RuntimeWarning(Warning): pass -from .object import object - class set(object): """ set() -> new empty set object @@ -4530,8 +4446,6 @@ class set(object): __hash__ = None -from .object import object - class slice(object): """ slice(stop) @@ -4607,8 +4521,6 @@ class slice(object): __hash__ = None -from .object import object - class staticmethod(object): """ staticmethod(function) -> method @@ -4648,7 +4560,16 @@ class staticmethod(object): __dict__ = None # (!) real value is '' -from .Exception import Exception +class StopAsyncIteration(Exception): + """ Signal the end from iterator.__anext__(). """ + def __init__(self, *args, **kwargs): # real signature unknown + pass + + @staticmethod # known case of __new__ + def __new__(*args, **kwargs): # real signature unknown + """ Create and return a new object. See help(type) for accurate signature. """ + pass + class StopIteration(Exception): """ Signal the end from iterator.__next__(). """ @@ -4660,8 +4581,6 @@ class StopIteration(Exception): -from .object import object - class str(object): """ str(object='') -> str @@ -5080,11 +4999,12 @@ class str(object): """ S.translate(table) -> str - Return a copy of the string S, where all characters have been mapped - through the given translation table, which must be a mapping of - Unicode ordinals to Unicode ordinals, strings, or None. - Unmapped characters are left untouched. Characters mapped to None - are deleted. + Return a copy of the string S in which each character has been mapped + through the given translation table. The table must implement + lookup/indexing via __getitem__, for instance a dictionary or list, + mapping Unicode ordinals to Unicode ordinals, strings, or None. If + this operation raises LookupError, the character is left untouched. + Characters mapped to None are deleted. """ return "" @@ -5218,8 +5138,6 @@ class str(object): pass -from .object import object - class super(object): """ super() -> same as super(__class__, ) @@ -5293,8 +5211,6 @@ class super(object): -from .Warning import Warning - class SyntaxWarning(Warning): """ Base class for warnings about dubious syntax. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5306,8 +5222,6 @@ class SyntaxWarning(Warning): pass -from .Exception import Exception - class SystemError(Exception): """ Internal error in the Python interpreter. @@ -5324,8 +5238,6 @@ class SystemError(Exception): pass -from .BaseException import BaseException - class SystemExit(BaseException): """ Request to exit from the interpreter. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5336,24 +5248,18 @@ class SystemExit(BaseException): -from .IndentationError import IndentationError - class TabError(IndentationError): """ Improper mixture of spaces and tabs. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .OSError import OSError - class TimeoutError(OSError): """ Timeout expired. """ def __init__(self, *args, **kwargs): # real signature unknown pass -from .object import object - class tuple(object): """ tuple() -> empty tuple @@ -5454,12 +5360,6 @@ class tuple(object): """ Return self*value. """ pass - def __sizeof__(self): # real signature unknown; restored from __doc__ - """ T.__sizeof__() -- size of T in memory, in bytes """ - pass - - -from .object import object class type(object): """ @@ -5554,11 +5454,11 @@ class type(object): object, ) __base__ = object - __basicsize__ = 412 - __dictoffset__ = 132 + __basicsize__ = 864 + __dictoffset__ = 264 __dict__ = None # (!) real value is '' - __flags__ = -2146675712 - __itemsize__ = 20 + __flags__ = 2148291584 + __itemsize__ = 40 __mro__ = ( None, # (!) forward: type, real value is '' object, @@ -5566,11 +5466,9 @@ class type(object): __name__ = 'type' __qualname__ = 'type' __text_signature__ = None - __weakrefoffset__ = 184 + __weakrefoffset__ = 368 -from .Exception import Exception - class TypeError(Exception): """ Inappropriate argument type. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5582,8 +5480,6 @@ class TypeError(Exception): pass -from .NameError import NameError - class UnboundLocalError(NameError): """ Local name referenced but not bound to a value. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5595,8 +5491,6 @@ class UnboundLocalError(NameError): pass -from .Exception import Exception - class ValueError(Exception): """ Inappropriate argument value (of correct type). """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5608,8 +5502,6 @@ class ValueError(Exception): pass -from .ValueError import ValueError - class UnicodeError(ValueError): """ Unicode related error. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5621,8 +5513,6 @@ class UnicodeError(ValueError): pass -from .UnicodeError import UnicodeError - class UnicodeDecodeError(UnicodeError): """ Unicode decoding error. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5654,8 +5544,6 @@ class UnicodeDecodeError(UnicodeError): -from .UnicodeError import UnicodeError - class UnicodeEncodeError(UnicodeError): """ Unicode encoding error. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5687,8 +5575,6 @@ class UnicodeEncodeError(UnicodeError): -from .UnicodeError import UnicodeError - class UnicodeTranslateError(UnicodeError): """ Unicode translation error. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5720,8 +5606,6 @@ class UnicodeTranslateError(UnicodeError): -from .Warning import Warning - class UnicodeWarning(Warning): """ Base class for warnings about Unicode related problems, mostly @@ -5736,8 +5620,6 @@ class UnicodeWarning(Warning): pass -from .Warning import Warning - class UserWarning(Warning): """ Base class for warnings generated by user code. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5749,8 +5631,6 @@ class UserWarning(Warning): pass -from .ArithmeticError import ArithmeticError - class ZeroDivisionError(ArithmeticError): """ Second argument to a division or modulo operation was zero. """ def __init__(self, *args, **kwargs): # real signature unknown @@ -5762,8 +5642,6 @@ class ZeroDivisionError(ArithmeticError): pass -from .object import object - class zip(object): """ zip(iter1 [,iter2 [...]]) --> zip object @@ -5798,8 +5676,6 @@ class zip(object): pass -from .object import object - class __loader__(object): """ Meta path import for built-in modules. @@ -5807,6 +5683,14 @@ class __loader__(object): All methods are either class or static methods to avoid the need to instantiate the class. """ + def create_module(self, *args, **kwargs): # real signature unknown + """ Create a built-in module """ + pass + + def exec_module(self, *args, **kwargs): # real signature unknown + """ Exec a built-in module """ + pass + def find_module(self, *args, **kwargs): # real signature unknown """ Find the built-in module. @@ -5833,7 +5717,11 @@ class __loader__(object): pass def load_module(self, *args, **kwargs): # real signature unknown - """ Load a built-in module. """ + """ + Load the specified module into sys.modules and return it. + + This method is deprecated. Use loader.exec_module instead. + """ pass def module_repr(module): # reliably restored by inspect From 538263246ae5ceffe33c8a6444943b007d3a2c9e Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Sat, 12 Sep 2015 13:59:48 +0300 Subject: [PATCH 12/15] Type inference for async functions and 'await' expressions (PY-16094) We had to make PyCollectionType look more like a parameterized type where you can specify several type parameters and they don't look like a single tuple parameter. Also we had to get rid of several (but not all yet) places where we made an implicit assumption that when you were interating over a PyCollectionType you could just yield its element type. --- .../src/com/jetbrains/python/PyNames.java | 1 + .../codeInsight/PyTypingTypeProvider.java | 25 ++-- .../documentation/PyTypeModelBuilder.java | 29 +++-- .../python/psi/impl/PyBuiltinCache.java | 35 ++++-- .../psi/impl/PyCallExpressionHelper.java | 4 +- .../python/psi/impl/PyFunctionImpl.java | 25 +++- .../psi/impl/PyGeneratorExpressionImpl.java | 9 +- .../psi/impl/PyListCompExpressionImpl.java | 4 +- .../python/psi/impl/PyNamedParameterImpl.java | 4 +- .../psi/impl/PyPrefixExpressionImpl.java | 56 ++++++++- .../impl/PySubscriptionExpressionImpl.java | 4 +- .../psi/impl/PyTargetExpressionImpl.java | 66 +++++----- .../python/psi/types/PyCollectionType.java | 8 +- .../psi/types/PyCollectionTypeImpl.java | 34 ++++-- .../python/psi/types/PyTypeChecker.java | 26 ++-- .../python/psi/types/PyTypeParser.java | 24 ++-- .../PyTypeCheckerInspection/Generator.py | 22 ++-- .../com/jetbrains/python/Py3TypeTest.java | 113 ++++++++++++++++++ .../jetbrains/python/PyTypeParserTest.java | 31 ++--- .../com/jetbrains/python/PyTypeTest.java | 57 +++++---- 20 files changed, 389 insertions(+), 188 deletions(-) create mode 100644 python/testSrc/com/jetbrains/python/Py3TypeTest.java diff --git a/python/psi-api/src/com/jetbrains/python/PyNames.java b/python/psi-api/src/com/jetbrains/python/PyNames.java index dd799b45f763..3d77078f7982 100644 --- a/python/psi-api/src/com/jetbrains/python/PyNames.java +++ b/python/psi-api/src/com/jetbrains/python/PyNames.java @@ -93,6 +93,7 @@ public class PyNames { public static final String FAKE_FUNCTION = "__function"; public static final String FAKE_METHOD = "__method"; public static final String FAKE_NAMEDTUPLE = "__namedtuple"; + public static final String FAKE_COROUTINE = "__coroutine"; public static final String FUTURE_MODULE = "__future__"; public static final String UNICODE_LITERALS = "unicode_literals"; diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index 5af330a71dc3..05920396d020 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -187,20 +187,9 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { final PyClass cls = function.getContainingClass(); if (cls != null) { final List genericTypes = collectGenericTypes(cls, context); - - final PyType elementType; - if (genericTypes.size() == 1) { - elementType = genericTypes.get(0); - } - else if (genericTypes.size() > 1) { - elementType = PyTupleType.create(cls, genericTypes.toArray(new PyType[genericTypes.size()])); - } - else { - elementType = null; - } - - if (elementType != null) { - return new PyCollectionTypeImpl(cls, false, elementType); + final List elementTypes = new ArrayList(genericTypes); + if (!elementTypes.isEmpty()) { + return new PyCollectionTypeImpl(cls, false, elementTypes); } } } @@ -438,6 +427,9 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { types.add(getType(expr, context)); } } + else if (indexExpr != null) { + types.add(getType(indexExpr, context)); + } return types; } @@ -450,13 +442,12 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { final PyType operandType = getType(operand, context); if (operandType instanceof PyClassType) { final PyClass cls = ((PyClassType)operandType).getPyClass(); + final List indexTypes = getIndexTypes(subscriptionExpr, context); if (PyNames.TUPLE.equals(cls.getQualifiedName())) { - final List indexTypes = getIndexTypes(subscriptionExpr, context); return PyTupleType.create(expression, indexTypes.toArray(new PyType[indexTypes.size()])); } else if (indexExpr != null) { - final PyType indexType = context.getType(indexExpr); - return new PyCollectionTypeImpl(cls, false, indexType); + return new PyCollectionTypeImpl(cls, false, indexTypes); } } } diff --git a/python/src/com/jetbrains/python/documentation/PyTypeModelBuilder.java b/python/src/com/jetbrains/python/documentation/PyTypeModelBuilder.java index ad689b66681c..8fdd9b0460c5 100644 --- a/python/src/com/jetbrains/python/documentation/PyTypeModelBuilder.java +++ b/python/src/com/jetbrains/python/documentation/PyTypeModelBuilder.java @@ -200,23 +200,22 @@ public class PyTypeModelBuilder { TypeModel result = null; if (type instanceof PyCollectionType) { final String name = type.getName(); - final PyType elementType = ((PyCollectionType)type).getElementType(myContext); - final List elementTypes = new ArrayList(); - if (elementType instanceof PyTupleType) { - final PyTupleType tupleType = (PyTupleType)elementType; - final int n = tupleType.getElementCount(); - for (int i = 0; i < n; i++) { - final PyType t = tupleType.getElementType(i); - if (t != null) { - elementTypes.add(build(t, true)); - } + final List elementTypes = ((PyCollectionType)type).getElementTypes(myContext); + boolean nullOnlyTypes = true; + for (PyType elementType : elementTypes) { + if (elementType != null) { + nullOnlyTypes = false; + break; } } - else if (elementType != null) { - elementTypes.add(build(elementType, true)); - } - if (!elementTypes.isEmpty()) { - result = new CollectionOf(name, elementTypes); + final List elementModels = new ArrayList(); + if (!nullOnlyTypes) { + for (PyType elementType : elementTypes) { + elementModels.add(build(elementType, true)); + } + if (!elementModels.isEmpty()) { + result = new CollectionOf(name, elementModels); + } } } else if (type instanceof PyUnionType && allowUnions) { diff --git a/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java b/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java index 0e58f56d5ff3..415fed8ba1c9 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java +++ b/python/src/com/jetbrains/python/psi/impl/PyBuiltinCache.java @@ -40,9 +40,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.File; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Provides access to Python builtins via skeletons. @@ -164,28 +162,39 @@ public class PyBuiltinCache { public PyType createLiteralCollectionType(final PySequenceExpression sequence, final String name, @NotNull TypeEvalContext context) { final PyClass cls = getClass(name); if (cls != null) { - return new PyCollectionTypeImpl(cls, false, getSequenceElementType(sequence, context)); + return new PyCollectionTypeImpl(cls, false, getSequenceElementTypes(sequence, context)); } return null; } - @Nullable - private static PyType getSequenceElementType(@NotNull PySequenceExpression sequence, @NotNull TypeEvalContext context) { + @NotNull + private static List getSequenceElementTypes(@NotNull PySequenceExpression sequence, @NotNull TypeEvalContext context) { final PyExpression[] elements = sequence.getElements(); if (elements.length == 0 || elements.length > 10 /* performance */) { - return null; + return Collections.singletonList(null); } - final PyType result = context.getType(elements[0]); - if (result == null) { - return null; + final PyType firstElementType = context.getType(elements[0]); + if (firstElementType == null) { + return Collections.singletonList(null); } for (int i = 1; i < elements.length; i++) { final PyType elementType = context.getType(elements[i]); - if (elementType == null || !elementType.equals(result)) { - return null; + if (elementType == null || !elementType.equals(firstElementType)) { + return Collections.singletonList(null); } } - return result; + if (sequence instanceof PyDictLiteralExpression) { + if (firstElementType instanceof PyTupleType) { + final PyTupleType tupleType = (PyTupleType)firstElementType; + if (tupleType.getElementCount() == 2) { + return Arrays.asList(tupleType.getElementType(0), tupleType.getElementType(1)); + } + } + return Arrays.asList(null, null); + } + else { + return Collections.singletonList(firstElementType); + } } @Nullable diff --git a/python/src/com/jetbrains/python/psi/impl/PyCallExpressionHelper.java b/python/src/com/jetbrains/python/psi/impl/PyCallExpressionHelper.java index 9588a385d2fd..59889b32c63d 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyCallExpressionHelper.java +++ b/python/src/com/jetbrains/python/psi/impl/PyCallExpressionHelper.java @@ -498,8 +498,8 @@ public class PyCallExpressionHelper { if (cls != null) { if (init.getContainingClass() != cls) { if (t instanceof PyCollectionType) { - final PyType elementType = ((PyCollectionType)t).getElementType(context); - return Ref.create(new PyCollectionTypeImpl(cls, false, elementType)); + final List elementTypes = ((PyCollectionType)t).getElementTypes(context); + return Ref.create(new PyCollectionTypeImpl(cls, false, elementTypes)); } return Ref.create(new PyClassTypeImpl(cls, false)); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index bab647903f24..05d2bf78fc77 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -204,6 +204,9 @@ public class PyFunctionImpl extends PyBaseElementImpl implements } return getReturnStatementType(context); } + else if (isAsync()) { + return createCoroutineType(null); + } return null; } @@ -319,7 +322,9 @@ public class PyFunctionImpl extends PyBaseElementImpl implements final PyType type = context.getType(node); if (node.isDelegating() && type instanceof PyCollectionType) { final PyCollectionType collectionType = (PyCollectionType)type; - types.add(collectionType.getElementType(context)); + // TODO: Select the parameter types that matches T in Iterable[T] + final List elementTypes = collectionType.getElementTypes(context); + types.add(elementTypes.isEmpty() ? null : elementTypes.get(0)); } else { types.add(type); @@ -341,7 +346,8 @@ public class PyFunctionImpl extends PyBaseElementImpl implements if (elementType != null) { final PyClass generator = cache.getClass(PyNames.FAKE_GENERATOR); if (generator != null) { - return Ref.create(new PyCollectionTypeImpl(generator, false, elementType.get())); + final List parameters = Arrays.asList(elementType.get(), null, getReturnStatementType(context)); + return Ref.create(new PyCollectionTypeImpl(generator, false, parameters)); } } if (!types.isEmpty()) { @@ -352,7 +358,7 @@ public class PyFunctionImpl extends PyBaseElementImpl implements @Nullable public PyType getReturnStatementType(TypeEvalContext typeEvalContext) { - ReturnVisitor visitor = new ReturnVisitor(this, typeEvalContext); + final ReturnVisitor visitor = new ReturnVisitor(this, typeEvalContext); final PyStatementList statements = getStatementList(); statements.accept(visitor); if (isGeneratedStub() && !visitor.myHasReturns) { @@ -361,7 +367,18 @@ public class PyFunctionImpl extends PyBaseElementImpl implements } return null; } - return visitor.result(); + final PyType returnType = visitor.result(); + if (isAsync()) { + return createCoroutineType(returnType); + } + return returnType; + } + + @Nullable + private PyType createCoroutineType(@Nullable PyType returnType) { + final PyBuiltinCache cache = PyBuiltinCache.getInstance(this); + final PyClass generator = cache.getClass(PyNames.FAKE_COROUTINE); + return generator != null ? new PyCollectionTypeImpl(generator, false, Collections.singletonList(returnType)) : null; } public PyFunction asMethod() { diff --git a/python/src/com/jetbrains/python/psi/impl/PyGeneratorExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyGeneratorExpressionImpl.java index 507a219a1bfc..5aac98489f25 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyGeneratorExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyGeneratorExpressionImpl.java @@ -19,13 +19,12 @@ import com.intellij.lang.ASTNode; import com.intellij.psi.PsiElement; import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; -import com.jetbrains.python.psi.types.PyCollectionTypeImpl; -import com.jetbrains.python.psi.types.PyType; -import com.jetbrains.python.psi.types.TypeEvalContext; +import com.jetbrains.python.psi.types.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -48,8 +47,8 @@ public class PyGeneratorExpressionImpl extends PyComprehensionElementImpl implem final PyBuiltinCache cache = PyBuiltinCache.getInstance(this); final PyClass generator = cache.getClass(PyNames.FAKE_GENERATOR); if (resultExpr != null && generator != null) { - final PyType elementType = context.getType(resultExpr); - return new PyCollectionTypeImpl(generator, false, elementType); + final List parameters = Arrays.asList(context.getType(resultExpr), null, PyNoneType.INSTANCE); + return new PyCollectionTypeImpl(generator, false, parameters); } return null; } diff --git a/python/src/com/jetbrains/python/psi/impl/PyListCompExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyListCompExpressionImpl.java index bf1bbed637f7..e836e2dcf163 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyListCompExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyListCompExpressionImpl.java @@ -26,6 +26,8 @@ import com.jetbrains.python.psi.types.TypeEvalContext; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.Collections; + /** * @author yole */ @@ -47,7 +49,7 @@ public class PyListCompExpressionImpl extends PyComprehensionElementImpl impleme final PyClass list = cache.getClass("list"); if (resultExpr != null && list != null) { final PyType elementType = context.getType(resultExpr); - return new PyCollectionTypeImpl(list, false, elementType); + return new PyCollectionTypeImpl(list, false, Collections.singletonList(elementType)); } return cache.getListType(); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java index cd604cf21500..3070f4c1ec7f 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java @@ -224,8 +224,8 @@ public class PyNamedParameterImpl extends PyBaseElementImpl elementTypes = ((PyCollectionType)initType).getElementTypes(context); + return new PyCollectionTypeImpl(containingClass, false, elementTypes); } } } diff --git a/python/src/com/jetbrains/python/psi/impl/PyPrefixExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyPrefixExpressionImpl.java index 177d0c40f651..96e78cc9b5da 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyPrefixExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyPrefixExpressionImpl.java @@ -27,11 +27,13 @@ import com.jetbrains.python.PythonDialectsTokenSetProvider; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.references.PyOperatorReference; import com.jetbrains.python.psi.resolve.PyResolveContext; -import com.jetbrains.python.psi.types.PyType; -import com.jetbrains.python.psi.types.TypeEvalContext; +import com.jetbrains.python.psi.types.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; +import java.util.List; + /** * @author yole */ @@ -81,11 +83,23 @@ public class PyPrefixExpressionImpl extends PyElementImpl implements PyPrefixExp if (getOperator() == PyTokenTypes.NOT_KEYWORD) { return PyBuiltinCache.getInstance(this).getBoolType(); } + final boolean isAwait = getOperator() == PyTokenTypes.AWAIT_KEYWORD; + if (isAwait) { + final PyExpression operand = getOperand(); + if (operand != null) { + final PyType operandType = context.getType(operand); + final PyType type = getGeneratorReturnType(operandType, context); + if (type != null) { + return type; + } + } + } final PsiReference ref = getReference(PyResolveContext.noImplicits().withTypeEvalContext(context)); final PsiElement resolved = ref.resolve(); if (resolved instanceof PyCallable) { // TODO: Make PyPrefixExpression a PyCallSiteExpression, use getCallType() here and analyze it in PyTypeChecker.analyzeCallSite() - return ((PyCallable)resolved).getReturnType(context, key); + final PyType returnType = ((PyCallable)resolved).getReturnType(context, key); + return isAwait ? getGeneratorReturnType(returnType, context) : returnType; } return null; } @@ -123,4 +137,40 @@ public class PyPrefixExpressionImpl extends PyElementImpl implements PyPrefixExp final PsiElement op = getPsiOperator(); return op != null ? op.getNode() : null; } + + @Nullable + private static PyType getGeneratorReturnType(@Nullable PyType type, @NotNull TypeEvalContext context) { + if (type instanceof PyClassLikeType) { + final PyClassLikeType classLikeType = (PyClassLikeType)type; + // TODO: Understand typing.Generator as well + final String classQName = classLikeType.getClassQName(); + if (PyNames.FAKE_GENERATOR.equals(classQName)) { + if (type instanceof PyCollectionType) { + final PyCollectionType collectionType = (PyCollectionType)type; + final List elementTypes = collectionType.getElementTypes(context); + if (elementTypes.size() == 3) { + return elementTypes.get(2); + } + } + } + else if (PyNames.FAKE_COROUTINE.equals(classQName)) { + if (type instanceof PyCollectionType) { + final PyCollectionType collectionType = (PyCollectionType)type; + final List elementTypes = collectionType.getElementTypes(context); + if (elementTypes.size() == 1) { + return elementTypes.get(0); + } + } + } + } + else if (type instanceof PyUnionType) { + final List memberReturnTypes = new ArrayList(); + final PyUnionType unionType = (PyUnionType)type; + for (PyType member : unionType.getMembers()) { + memberReturnTypes.add(getGeneratorReturnType(member, context)); + } + return PyUnionType.union(memberReturnTypes); + } + return null; + } } diff --git a/python/src/com/jetbrains/python/psi/impl/PySubscriptionExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PySubscriptionExpressionImpl.java index a882b31633e6..a50bdf3b2d5b 100644 --- a/python/src/com/jetbrains/python/psi/impl/PySubscriptionExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PySubscriptionExpressionImpl.java @@ -88,7 +88,9 @@ public class PySubscriptionExpressionImpl extends PyElementImpl implements PySub res = ((PySubscriptableType)type).getElementType(indexExpression, context); } else if (type instanceof PyCollectionType) { - res = ((PyCollectionType)type).getElementType(context); + // TODO: Select the parameter type that matches T in Iterable[T] + final List elementTypes = ((PyCollectionType)type).getElementTypes(context); + res = elementTypes.isEmpty() ? null : elementTypes.get(0); } } } diff --git a/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java index 43c15943f75f..249e853ffcd0 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java @@ -346,20 +346,7 @@ public class PyTargetExpressionImpl extends PyBaseElementImpl memberTypes = new ArrayList(); for (int i = 0; i < tupleType.getElementCount(); i++) { @@ -377,30 +364,45 @@ public class PyTargetExpressionImpl extends PyBaseElementImpl elementTypes = ((PyCollectionType)type).getElementTypes(context); + // TODO: Select the parameter type that matches T in Iterable[T] + return elementTypes.isEmpty() ? null : elementTypes.get(0); + } + return null; } @Nullable diff --git a/python/src/com/jetbrains/python/psi/types/PyCollectionType.java b/python/src/com/jetbrains/python/psi/types/PyCollectionType.java index 7df9aea7f48a..3da104dee6fe 100644 --- a/python/src/com/jetbrains/python/psi/types/PyCollectionType.java +++ b/python/src/com/jetbrains/python/psi/types/PyCollectionType.java @@ -15,12 +15,14 @@ */ package com.jetbrains.python.psi.types; -import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.NotNull; + +import java.util.List; /** * @author yole */ public interface PyCollectionType extends PyType { - @Nullable - PyType getElementType(TypeEvalContext context); + @NotNull + List getElementTypes(@NotNull TypeEvalContext context); } diff --git a/python/src/com/jetbrains/python/psi/types/PyCollectionTypeImpl.java b/python/src/com/jetbrains/python/psi/types/PyCollectionTypeImpl.java index 9b3534a40f01..ecefd2c465f4 100644 --- a/python/src/com/jetbrains/python/psi/types/PyCollectionTypeImpl.java +++ b/python/src/com/jetbrains/python/psi/types/PyCollectionTypeImpl.java @@ -21,30 +21,33 @@ import com.jetbrains.python.psi.stubs.PyClassNameIndex; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.List; + /** * @author yole */ public class PyCollectionTypeImpl extends PyClassTypeImpl implements PyCollectionType { - private final PyType myElementType; + @NotNull private final List myElementTypes; - public PyCollectionTypeImpl(@NotNull PyClass source, boolean isDefinition, PyType elementType) { + public PyCollectionTypeImpl(@NotNull PyClass source, boolean isDefinition, @NotNull List elementTypes) { super(source, isDefinition); - myElementType = elementType; + myElementTypes = elementTypes; } + @NotNull @Override - public PyType getElementType(TypeEvalContext context) { - return myElementType; + public List getElementTypes(@NotNull TypeEvalContext context) { + return myElementTypes; } @Nullable public static PyCollectionTypeImpl createTypeByQName(@NotNull Project project, String classQualifiedName, boolean isDefinition, - PyType elementType) { - PyClass pyClass = PyClassNameIndex.findClass(classQualifiedName, project); + @NotNull List elementTypes) { + final PyClass pyClass = PyClassNameIndex.findClass(classQualifiedName, project); if (pyClass == null) { return null; } - return new PyCollectionTypeImpl(pyClass, isDefinition, elementType); + return new PyCollectionTypeImpl(pyClass, isDefinition, elementTypes); } @Override @@ -56,15 +59,24 @@ public class PyCollectionTypeImpl extends PyClassTypeImpl implements PyCollectio PyCollectionType type = (PyCollectionType)o; final TypeEvalContext context = TypeEvalContext.codeInsightFallback(myClass.getProject()); - if (myElementType != null ? !myElementType.equals(type.getElementType(context)) : type.getElementType(context) != null) return false; - + final List otherElementTypes = type.getElementTypes(context); + if (myElementTypes.size() != otherElementTypes.size()) return false; + for (int i = 0; i < myElementTypes.size(); i++) { + final PyType elementType = myElementTypes.get(i); + final PyType otherElementType = otherElementTypes.get(i); + if (elementType == null && otherElementType != null) return false; + if (elementType != null && !elementType.equals(otherElementType)) return false; + } return true; } @Override public int hashCode() { int result = super.hashCode(); - result = 31 * result + (myElementType != null ? myElementType.hashCode() : 0); + result = 31 * result; + for (PyType type : myElementTypes) { + result += type != null ? type.hashCode() : 0; + } return result; } } diff --git a/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java b/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java index fd8320d1f93f..5a32a1f07d6d 100644 --- a/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java +++ b/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java @@ -123,9 +123,16 @@ public class PyTypeChecker { if (!matchClasses(superClass, subClass, context)) { return false; } - final PyType superElementType = ((PyCollectionType)expected).getElementType(context); - final PyType subElementType = ((PyCollectionType)actual).getElementType(context); - return match(superElementType, subElementType, context, substitutions, recursive); + // TODO: Match generic parameters based on the correspondence between the generic parameters of subClass and its base classes + final List superElementTypes = ((PyCollectionType)expected).getElementTypes(context); + final List subElementTypes = ((PyCollectionType)actual).getElementTypes(context); + for (int i = 0; i < subElementTypes.size(); i++) { + final PyType superElementType = i < superElementTypes.size() ? superElementTypes.get(i) : null; + if (!match(superElementType, subElementTypes.get(i), context, substitutions, recursive)) { + return false; + } + } + return true; } else if (expected instanceof PyTupleType && actual instanceof PyTupleType) { final PyTupleType superTupleType = (PyTupleType)expected; @@ -311,7 +318,9 @@ public class PyTypeChecker { } else if (type instanceof PyCollectionType) { final PyCollectionType collection = (PyCollectionType)type; - collectGenerics(collection.getElementType(context), context, collected, visited); + for (PyType elementType : collection.getElementTypes(context)) { + collectGenerics(elementType, context, collected, visited); + } } else if (type instanceof PyTupleType) { final PyTupleType tuple = (PyTupleType)type; @@ -352,9 +361,12 @@ public class PyTypeChecker { } else if (type instanceof PyCollectionTypeImpl) { final PyCollectionTypeImpl collection = (PyCollectionTypeImpl)type; - final PyType elem = collection.getElementType(context); - final PyType subst = substitute(elem, substitutions, context); - return new PyCollectionTypeImpl(collection.getPyClass(), collection.isDefinition(), subst); + final List elementTypes = collection.getElementTypes(context); + final List substitutes = new ArrayList(); + for (PyType elementType : elementTypes) { + substitutes.add(substitute(elementType, substitutions, context)); + } + return new PyCollectionTypeImpl(collection.getPyClass(), collection.isDefinition(), substitutes); } else if (type instanceof PyTupleType) { final PyTupleType tuple = (PyTupleType)type; diff --git a/python/src/com/jetbrains/python/psi/types/PyTypeParser.java b/python/src/com/jetbrains/python/psi/types/PyTypeParser.java index e10966c07019..073b9f5dbb04 100644 --- a/python/src/com/jetbrains/python/psi/types/PyTypeParser.java +++ b/python/src/com/jetbrains/python/psi/types/PyTypeParser.java @@ -39,10 +39,7 @@ import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.io.StringReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.jetbrains.python.psi.types.PyTypeTokenTypes.IDENTIFIER; import static com.jetbrains.python.psi.types.PyTypeTokenTypes.PARAMETER; @@ -188,17 +185,16 @@ public class PyTypeParser { final List third = value.getSecond(); final PyType firstType = first.getType(); if (firstType instanceof PyClassType) { - final List tupleTypes = new ArrayList(); - tupleTypes.add(second.getType()); + final List typesInBrackets = new ArrayList(); + typesInBrackets.add(second.getType()); ParseResult result = first; result = result.merge(second); for (ParseResult r : third) { - tupleTypes.add(r.getType()); + typesInBrackets.add(r.getType()); result = result.merge(r); } - final PyType elementType = third.isEmpty() ? second.getType() : - PyTupleType.create(anchor, tupleTypes.toArray(new PyType[tupleTypes.size()])); - final PyCollectionTypeImpl type = new PyCollectionTypeImpl(((PyClassType)firstType).getPyClass(), false, elementType); + final List elementTypes = third.isEmpty() ? Collections.singletonList(second.getType()) : typesInBrackets; + final PyCollectionTypeImpl type = new PyCollectionTypeImpl(((PyClassType)firstType).getPyClass(), false, elementTypes); return result.withType(type); } return EMPTY_RESULT; @@ -215,7 +211,8 @@ public class PyTypeParser { final PyType secondType = secondResult.getType(); if (firstType != null) { if (firstType instanceof PyClassType && secondType != null) { - return result.withType(new PyCollectionTypeImpl(((PyClassType)firstType).getPyClass(), false, secondType)); + return result.withType(new PyCollectionTypeImpl(((PyClassType)firstType).getPyClass(), false, + Collections.singletonList(secondType))); } return result.withType(firstType); } @@ -232,8 +229,9 @@ public class PyTypeParser { final ParseResult third = value.getSecond(); final PyType firstType = first.getType(); if (firstType instanceof PyClassType) { - final PyTupleType tupleType = PyTupleType.create(anchor, new PyType[]{second.getType(), third.getType()}); - final PyCollectionTypeImpl type = new PyCollectionTypeImpl(((PyClassType)firstType).getPyClass(), false, tupleType); + final List elementTypes = Arrays.asList(second.getType(), third.getType()); + final PyCollectionTypeImpl type = new PyCollectionTypeImpl(((PyClassType)firstType).getPyClass(), false, + elementTypes); return first.merge(second).merge(third).withType(type); } return EMPTY_RESULT; diff --git a/python/testData/inspections/PyTypeCheckerInspection/Generator.py b/python/testData/inspections/PyTypeCheckerInspection/Generator.py index 95481a651b42..ac43b303784e 100644 --- a/python/testData/inspections/PyTypeCheckerInspection/Generator.py +++ b/python/testData/inspections/PyTypeCheckerInspection/Generator.py @@ -79,19 +79,19 @@ def test(): return xs return [ ''.join(gen(10)), - f_1(gen(11)), - f_2(gen(11)), - f_3(gen(11)), - f_4(gen(11)), - f_5(gen(11)), - f_6(gen(11)), - f_7(gen(11)), - f_8(gen(11)), + f_1(gen(11)), + f_2(gen(11)), + f_3(gen(11)), + f_4(gen(11)), + f_5(gen(11)), + f_6(gen(11)), + f_7(gen(11)), + f_8(gen(11)), f_9(gen(11)), f_10(gen(11)), - f_11(gen(11)), - f_12(gen(11)), - f_13(gen(11)), + f_11(gen(11)), + f_12(gen(11)), + f_13(gen(11)), f_14(gen(11)), f_15(gen(11)), f_15('foo'.split('o')), diff --git a/python/testSrc/com/jetbrains/python/Py3TypeTest.java b/python/testSrc/com/jetbrains/python/Py3TypeTest.java new file mode 100644 index 000000000000..22d97527143b --- /dev/null +++ b/python/testSrc/com/jetbrains/python/Py3TypeTest.java @@ -0,0 +1,113 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python; + +import com.intellij.testFramework.LightProjectDescriptor; +import com.jetbrains.python.documentation.PythonDocumentationProvider; +import com.jetbrains.python.fixtures.PyTestCase; +import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.psi.PyExpression; +import com.jetbrains.python.psi.types.PyType; +import com.jetbrains.python.psi.types.TypeEvalContext; + +/** + * @author vlan + */ +public class Py3TypeTest extends PyTestCase { + public static final String TEST_DIRECTORY = "/types/"; + + @Override + protected LightProjectDescriptor getProjectDescriptor() { + return ourPy3Descriptor; + } + + // PY-6702 + public void testYieldFromType() { + runWithLanguageLevel(LanguageLevel.PYTHON33, new Runnable() { + @Override + public void run() { + doTest("Union[str, int, float]", + "def subgen():\n" + + " for i in [1, 2, 3]:\n" + + " yield i\n" + + "\n" + + "def gen():\n" + + " yield 'foo'\n" + + " yield from subgen()\n" + + " yield 3.14\n" + + "\n" + + "for expr in gen():\n" + + " pass\n"); + } + }); + } + + public void testAwaitAwaitable() { + runWithLanguageLevel(LanguageLevel.PYTHON35, new Runnable() { + @Override + public void run() { + doTest("int", + "class C:\n" + + " def __await__(self):\n" + + " yield 'foo'\n" + + " return 0\n" + + "\n" + + "async def foo():\n" + + " c = C()\n" + + " expr = await c\n"); + } + }); + } + + public void testAsyncDefReturnType() { + runWithLanguageLevel(LanguageLevel.PYTHON35, new Runnable() { + @Override + public void run() { + doTest("__coroutine[int]", + "async def foo(x):\n" + + " await x\n" + + " return 0\n" + + "\n" + + "def bar(y):\n" + + " expr = foo(y)\n"); + } + }); + } + + public void testAwaitCoroutine() { + runWithLanguageLevel(LanguageLevel.PYTHON35, new Runnable() { + @Override + public void run() { + doTest("int", + "async def foo(x):\n" + + " await x\n" + + " return 0\n" + + "\n" + + "async def bar(y):\n" + + " expr = await foo(y)\n"); + } + }); + } + + private void doTest(final String expectedType, final String text) { + myFixture.configureByText(PythonFileType.INSTANCE, text); + final PyExpression expr = myFixture.findElementByText("expr", PyExpression.class); + final TypeEvalContext context = TypeEvalContext.userInitiated(expr.getProject(), expr.getContainingFile()).withTracing(); + final PyType actual = context.getType(expr); + final String actualType = PythonDocumentationProvider.getTypeName(actual, context); + assertEquals(expectedType, actualType); + } +} diff --git a/python/testSrc/com/jetbrains/python/PyTypeParserTest.java b/python/testSrc/com/jetbrains/python/PyTypeParserTest.java index a74b75806b8b..91fa46fbe2ed 100644 --- a/python/testSrc/com/jetbrains/python/PyTypeParserTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypeParserTest.java @@ -50,8 +50,9 @@ public class PyTypeParserTest extends PyTestCase { public void testListType() { myFixture.configureByFile("typeParser/typeParser.py"); final PyCollectionType type = (PyCollectionType) PyTypeParser.getTypeByName(myFixture.getFile(), "list of MyObject"); + assertNotNull(type); assertClassType(type, "list"); - assertClassType(type.getElementType(getTypeEvalContext()), "MyObject"); + assertClassType(type.getElementTypes(getTypeEvalContext()).get(0), "MyObject"); } public void testDictType() { @@ -59,12 +60,9 @@ public class PyTypeParserTest extends PyTestCase { final PyCollectionType type = (PyCollectionType) PyTypeParser.getTypeByName(myFixture.getFile(), "dict from str to MyObject"); assertNotNull(type); assertClassType(type, "dict"); - final PyType elementType = type.getElementType(getTypeEvalContext()); - assertInstanceOf(elementType, PyTupleType.class); - final PyTupleType tupleType = (PyTupleType)elementType; - assertEquals(2, tupleType.getElementCount()); - assertClassType(tupleType.getElementType(0), "str"); - assertClassType(tupleType.getElementType(1), "MyObject"); + final List elementTypes = type.getElementTypes(getTypeEvalContext()); + assertClassType(elementTypes.get(0), "str"); + assertClassType(elementTypes.get(1), "MyObject"); } private TypeEvalContext getTypeEvalContext() { @@ -182,8 +180,8 @@ public class PyTypeParserTest extends PyTestCase { final PyCollectionType collectionType = (PyCollectionType)type; assertNotNull(collectionType); assertEquals("list", collectionType.getName()); - final PyType elementType = collectionType.getElementType(TypeEvalContext.codeInsightFallback(null)); - assertInstanceOf(elementType, PyUnionType.class); + final List elementTypes = collectionType.getElementTypes(TypeEvalContext.codeInsightFallback(null)); + assertInstanceOf(elementTypes.get(0), PyUnionType.class); } public void testBoundedGeneric() { @@ -203,9 +201,8 @@ public class PyTypeParserTest extends PyTestCase { final PyCollectionType collectionType = (PyCollectionType)type; assertNotNull(collectionType); assertEquals("list", collectionType.getName()); - final PyType elementType = collectionType.getElementType(TypeEvalContext.codeInsightFallback(null)); - assertNotNull(elementType); - assertEquals("int", elementType.getName()); + final List elementTypes = collectionType.getElementTypes(TypeEvalContext.codeInsightFallback(null)); + assertEquals("int", elementTypes.get(0).getName()); } public void testBracketMultipleParams() { @@ -215,14 +212,12 @@ public class PyTypeParserTest extends PyTestCase { final PyCollectionType collectionType = (PyCollectionType)type; assertNotNull(collectionType); assertEquals("dict", collectionType.getName()); - final PyType elementType = collectionType.getElementType(TypeEvalContext.codeInsightFallback(null)); - assertNotNull(elementType); - assertInstanceOf(elementType, PyTupleType.class); - final PyTupleType tupleType = (PyTupleType)elementType; - final PyType first = tupleType.getElementType(0); + final List elementTypes = collectionType.getElementTypes(TypeEvalContext.codeInsightFallback(null)); + assertEquals(2, elementTypes.size()); + final PyType first = elementTypes.get(0); assertNotNull(first); assertEquals("str", first.getName()); - final PyType second = tupleType.getElementType(1); + final PyType second = elementTypes.get(1); assertNotNull(second); assertEquals("int", second.getName()); } diff --git a/python/testSrc/com/jetbrains/python/PyTypeTest.java b/python/testSrc/com/jetbrains/python/PyTypeTest.java index 34ff968711fe..32a318c62c9d 100644 --- a/python/testSrc/com/jetbrains/python/PyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypeTest.java @@ -23,6 +23,8 @@ import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher; import com.jetbrains.python.psi.types.*; import org.jetbrains.annotations.NotNull; +import java.util.List; + /** * @author yole */ @@ -357,28 +359,6 @@ public class PyTypeTest extends PyTestCase { assertNull(actual); } - // PY-6702 - public void testYieldFromType() { - PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), LanguageLevel.PYTHON33); - try { - doTest("Union[str, int, float]", - "def subgen():\n" + - " for i in [1, 2, 3]:\n" + - " yield i\n" + - "\n" + - "def gen():\n" + - " yield 'foo'\n" + - " yield from subgen()\n" + - " yield 3.14\n" + - "\n" + - "for expr in gen():\n" + - " pass\n"); - } - finally { - PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), null); - } - } - public void testFunctionAssignment() { doTest("int", "def f():\n" + @@ -475,6 +455,15 @@ public class PyTypeTest extends PyTestCase { "expr = f().next()\n"); } + public void testGeneratorFunctionType() { + doTest("__generator[str, Any, int]", + "def f():\n" + + " yield 'foo'\n" + + " return 0\n" + + "\n" + + "expr = f()\n"); + } + // PY-7020 public void testListComprehensionType() { final PyExpression expr = parseExpr("expr = [str(x) for x in range(10)]\n"); @@ -482,11 +471,10 @@ public class PyTypeTest extends PyTestCase { final PyType type = context.getType(expr); assertNotNull(type); assertInstanceOf(type, PyCollectionType.class); - assertEquals(type.getName(), "list"); + assertEquals("list", type.getName()); final PyCollectionType collectionType = (PyCollectionType)type; - final PyType elementType = collectionType.getElementType(context); - assertNotNull(elementType); - assertEquals(elementType.getName(), "str"); + final List elementTypes = collectionType.getElementTypes(context); + assertEquals("str", elementTypes.get(0).getName()); } // PY-7021 @@ -496,11 +484,20 @@ public class PyTypeTest extends PyTestCase { final PyType type = context.getType(expr); assertNotNull(type); assertInstanceOf(type, PyCollectionType.class); - assertEquals(type.getName(), "__generator"); + assertEquals("__generator", type.getName()); final PyCollectionType collectionType = (PyCollectionType)type; - final PyType elementType = collectionType.getElementType(context); - assertNotNull(elementType); - assertEquals(elementType.getName(), "str"); + final List elementTypes = collectionType.getElementTypes(context); + assertEquals("str", elementTypes.get(0).getName()); + assertTrue(PyTypeChecker.isUnknown(elementTypes.get(1))); + assertEquals("None", elementTypes.get(2).getName()); + } + + // PY-7021 + public void testIterOverGeneratorComprehension() { + doTest("str", + "xs = (str(x) for x in range(10))\n" + + "for expr in xs:\n" + + " pass\n"); } // EA-40207 From 17db5621d6e406db3ab9b985ec2b86defde5bb8e Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Sat, 12 Sep 2015 14:30:18 +0300 Subject: [PATCH 13/15] Wrap the return type from type providers for async function into the coroutine type (PY-16094) It is suggested in https://github.com/ambv/typehinting/issues/119 and most likely will become a part of PEP 484 after Python 3.5. --- .../python/psi/impl/PyFunctionImpl.java | 18 ++++++++++-------- .../com/jetbrains/python/Py3TypeTest.java | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index 05d2bf78fc77..8b8422be13a9 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -182,6 +182,12 @@ public class PyFunctionImpl extends PyBaseElementImpl implements @Nullable @Override public PyType getReturnType(@NotNull TypeEvalContext context, @NotNull TypeEvalContext.Key key) { + final PyType type = getReturnType(context); + return isAsync() ? createCoroutineType(type) : type; + } + + @Nullable + private PyType getReturnType(@NotNull TypeEvalContext context) { for (PyTypeProvider typeProvider : Extensions.getExtensions(PyTypeProvider.EP_NAME)) { final Ref returnTypeRef = typeProvider.getReturnType(this, context); if (returnTypeRef != null) { @@ -204,9 +210,6 @@ public class PyFunctionImpl extends PyBaseElementImpl implements } return getReturnStatementType(context); } - else if (isAsync()) { - return createCoroutineType(null); - } return null; } @@ -367,16 +370,15 @@ public class PyFunctionImpl extends PyBaseElementImpl implements } return null; } - final PyType returnType = visitor.result(); - if (isAsync()) { - return createCoroutineType(returnType); - } - return returnType; + return visitor.result(); } @Nullable private PyType createCoroutineType(@Nullable PyType returnType) { final PyBuiltinCache cache = PyBuiltinCache.getInstance(this); + if (returnType instanceof PyClassLikeType && PyNames.FAKE_COROUTINE.equals(((PyClassLikeType)returnType).getClassQName())) { + return returnType; + } final PyClass generator = cache.getClass(PyNames.FAKE_COROUTINE); return generator != null ? new PyCollectionTypeImpl(generator, false, Collections.singletonList(returnType)) : null; } diff --git a/python/testSrc/com/jetbrains/python/Py3TypeTest.java b/python/testSrc/com/jetbrains/python/Py3TypeTest.java index 22d97527143b..44d84c9bfdc6 100644 --- a/python/testSrc/com/jetbrains/python/Py3TypeTest.java +++ b/python/testSrc/com/jetbrains/python/Py3TypeTest.java @@ -102,6 +102,21 @@ public class Py3TypeTest extends PyTestCase { }); } + // Not in PEP 484 as for now, see https://github.com/ambv/typehinting/issues/119 + public void testCoroutineReturnTypeAnnotation() { + runWithLanguageLevel(LanguageLevel.PYTHON35, new Runnable() { + @Override + public void run() { + doTest("int", + "async def foo() -> int: ...\n" + + "\n" + + "async def bar():\n" + + " expr = await foo()\n"); + } + }); + + } + private void doTest(final String expectedType, final String text) { myFixture.configureByText(PythonFileType.INSTANCE, text); final PyExpression expr = myFixture.findElementByText("expr", PyExpression.class); From 9d2bd32bba2313167ee4849cd2158bd59ce34a32 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Sat, 12 Sep 2015 15:54:47 +0300 Subject: [PATCH 14/15] Extract async function (PY-16094) --- .../codeFragment/PyCodeFragment.java | 9 +++- .../codeFragment/PyCodeFragmentUtil.java | 4 +- .../python/psi/impl/PyFunctionBuilder.java | 9 ++++ .../extractmethod/PyExtractMethodUtil.java | 50 +++++++++++++++---- .../extractmethod/AsyncDef.after.py | 8 +++ .../extractmethod/AsyncDef.before.py | 3 ++ .../extractmethod/AwaitExpression.after.py | 7 +++ .../extractmethod/AwaitExpression.before.py | 3 ++ .../refactoring/PyExtractMethodTest.java | 8 +++ 9 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 python/testData/refactoring/extractmethod/AsyncDef.after.py create mode 100644 python/testData/refactoring/extractmethod/AsyncDef.before.py create mode 100644 python/testData/refactoring/extractmethod/AwaitExpression.after.py create mode 100644 python/testData/refactoring/extractmethod/AwaitExpression.before.py diff --git a/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragment.java b/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragment.java index c39a1f5e42c3..bb5d9eefdb74 100644 --- a/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragment.java +++ b/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragment.java @@ -26,17 +26,20 @@ public class PyCodeFragment extends CodeFragment { private final Set myGlobalWrites; private final Set myNonlocalWrites; private final boolean myYieldInside; + private final boolean myAsync; public PyCodeFragment(final Set input, final Set output, final Set globalWrites, final Set nonlocalWrites, final boolean returnInside, - final boolean yieldInside) { + final boolean yieldInside, + final boolean isAsync) { super(input, output, returnInside); myGlobalWrites = globalWrites; myNonlocalWrites = nonlocalWrites; myYieldInside = yieldInside; + myAsync = isAsync; } public Set getGlobalWrites() { @@ -50,4 +53,8 @@ public class PyCodeFragment extends CodeFragment { public boolean isYieldInside() { return myYieldInside; } + + public boolean isAsync() { + return myAsync; + } } diff --git a/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragmentUtil.java b/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragmentUtil.java index ca97e7b4e213..eb1f6a0f05e8 100644 --- a/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragmentUtil.java +++ b/python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragmentUtil.java @@ -99,13 +99,13 @@ public class PyCodeFragmentUtil { } } - final boolean yieldsFound = subGraphAnalysis.yieldExpressions > 0; if (yieldsFound && LanguageLevel.forElement(owner).isOlderThan(LanguageLevel.PYTHON33)) { throw new CannotCreateCodeFragmentException(PyBundle.message("refactoring.extract.method.error.yield")); } + final boolean isAsync = owner instanceof PyFunction && ((PyFunction)owner).isAsync(); - return new PyCodeFragment(inputNames, outputNames, globalWrites, nonlocalWrites, subGraphAnalysis.returns > 0, yieldsFound); + return new PyCodeFragment(inputNames, outputNames, globalWrites, nonlocalWrites, subGraphAnalysis.returns > 0, yieldsFound, isAsync); } private static boolean resolvesToBoundMethodParameter(@NotNull PsiElement element) { diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionBuilder.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionBuilder.java index 4a7abae08a89..8be934854256 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionBuilder.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionBuilder.java @@ -44,6 +44,7 @@ public class PyFunctionBuilder { private String[] myDocStringLines = null; @NotNull private final Map myDecoratorValues = new HashMap(); + private boolean myAsync = false; /** * Creates builder copying signature and doc from another one. @@ -136,6 +137,11 @@ public class PyFunctionBuilder { return this; } + public PyFunctionBuilder makeAsync() { + myAsync = true; + return this; + } + public PyFunctionBuilder statement(String text) { myStatements.add(text); return this; @@ -166,6 +172,9 @@ public class PyFunctionBuilder { } decoratorAppender.append("\n"); } + if (myAsync) { + builder.append("async "); + } builder.append("def "); builder.append(myName).append("("); builder.append(StringUtil.join(myParameters, ", ")); diff --git a/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java b/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java index a91efa182050..23b4243aee7d 100644 --- a/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java +++ b/python/src/com/jetbrains/python/refactoring/extractmethod/PyExtractMethodUtil.java @@ -117,6 +117,11 @@ public class PyExtractMethodUtil { .refactoringStarted(getRefactoringId(), beforeData); final StringBuilder builder = new StringBuilder(); + final boolean isAsync = fragment.isAsync(); + if (isAsync) { + builder.append("async "); + } + builder.append("def f():\n "); final List newMethodElements = new ArrayList(elementsRange); final boolean hasOutputVariables = !fragment.getOutputVariables().isEmpty(); @@ -124,14 +129,17 @@ public class PyExtractMethodUtil { final LanguageLevel languageLevel = LanguageLevel.forElement(statement1); if (hasOutputVariables) { // Generate return modified variables statements - StringUtil.join(fragment.getOutputVariables(), ", ", builder); + final String outputVariables = StringUtil.join(fragment.getOutputVariables(), ", "); + String newMethodText = builder + "return " + outputVariables; + builder.append(outputVariables); - final PsiElement returnStatement = generator.createFromText(languageLevel, PyElement.class, "return " + builder.toString()); + final PyFunction function = generator.createFromText(languageLevel, PyFunction.class, newMethodText); + final PsiElement returnStatement = function.getStatementList().getStatements()[0]; newMethodElements.add(returnStatement); } // Generate method - PyFunction generatedMethod = generateMethodFromElements(project, methodName, variableData, newMethodElements, flags); + PyFunction generatedMethod = generateMethodFromElements(project, methodName, variableData, newMethodElements, flags, isAsync); generatedMethod = insertGeneratedMethod(statement1, generatedMethod); // Process parameters @@ -148,7 +156,10 @@ public class PyExtractMethodUtil { else if (fragment.isReturnInstructionInside()) { builder.append("return "); } - if (fragment.isYieldInside()) { + if (isAsync) { + builder.append("await "); + } + else if (fragment.isYieldInside()) { builder.append("yield from "); } if (isMethod) { @@ -156,7 +167,8 @@ public class PyExtractMethodUtil { } builder.append(methodName).append("("); builder.append(createCallArgsString(variableData)).append(")"); - PsiElement callElement = generator.createFromText(languageLevel, PyElement.class, builder.toString()); + final PyFunction function = generator.createFromText(languageLevel, PyFunction.class, builder.toString()); + PsiElement callElement = function.getStatementList().getStatements()[0]; // replace statements with call callElement = replaceElements(elementsRange, callElement); @@ -297,7 +309,8 @@ public class PyExtractMethodUtil { @Override public void run() { // Generate method - PyFunction generatedMethod = generateMethodFromExpression(project, methodName, variableData, expression, flags); + final boolean isAsync = fragment.isAsync(); + PyFunction generatedMethod = generateMethodFromExpression(project, methodName, variableData, expression, flags, isAsync); generatedMethod = insertGeneratedMethod(expression, generatedMethod); // Process parameters @@ -306,7 +319,14 @@ public class PyExtractMethodUtil { // Generating call element final StringBuilder builder = new StringBuilder(); - if (fragment.isYieldInside()) { + if (isAsync) { + builder.append("async "); + } + builder.append("def f():\n "); + if (isAsync) { + builder.append("await "); + } + else if (fragment.isYieldInside()) { builder.append("yield from "); } else { @@ -318,8 +338,9 @@ public class PyExtractMethodUtil { builder.append(methodName); builder.append("(").append(createCallArgsString(variableData)).append(")"); final PyElementGenerator generator = PyElementGenerator.getInstance(project); - final PyElement generated = - generator.createFromText(LanguageLevel.forElement(expression), PyElement.class, builder.toString()); + final PyFunction function = generator.createFromText(LanguageLevel.forElement(expression), PyFunction.class, + builder.toString()); + final PyElement generated = function.getStatementList().getStatements()[0]; PsiElement callElement = null; if (generated instanceof PyReturnStatement) { callElement = ((PyReturnStatement)generated).getExpression(); @@ -495,10 +516,13 @@ public class PyExtractMethodUtil { @NotNull final String methodName, @NotNull final AbstractVariableData[] variableData, @NotNull final PsiElement expression, - @Nullable final PyUtil.MethodFlags flags) { + @Nullable final PyUtil.MethodFlags flags, boolean isAsync) { final PyFunctionBuilder builder = new PyFunctionBuilder(methodName); addDecorators(builder, flags); addFakeParameters(builder, variableData); + if (isAsync) { + builder.makeAsync(); + } final String text; if (expression instanceof PyYieldExpression) { text = String.format("(%s)", expression.getText()); @@ -515,10 +539,14 @@ public class PyExtractMethodUtil { @NotNull final String methodName, @NotNull final AbstractVariableData[] variableData, @NotNull final List elementsRange, - @Nullable PyUtil.MethodFlags flags) { + @Nullable PyUtil.MethodFlags flags, + boolean isAsync) { assert !elementsRange.isEmpty() : "Empty statements list was selected!"; final PyFunctionBuilder builder = new PyFunctionBuilder(methodName); + if (isAsync) { + builder.makeAsync(); + } addDecorators(builder, flags); addFakeParameters(builder, variableData); final PyFunction method = builder.buildFunction(project, LanguageLevel.forElement(elementsRange.get(0))); diff --git a/python/testData/refactoring/extractmethod/AsyncDef.after.py b/python/testData/refactoring/extractmethod/AsyncDef.after.py new file mode 100644 index 000000000000..87a5afc9b77f --- /dev/null +++ b/python/testData/refactoring/extractmethod/AsyncDef.after.py @@ -0,0 +1,8 @@ +async def foo(x): + y = await bar(x) + return await y + + +async def bar(x_new): + y = await x_new + return y diff --git a/python/testData/refactoring/extractmethod/AsyncDef.before.py b/python/testData/refactoring/extractmethod/AsyncDef.before.py new file mode 100644 index 000000000000..e655232e4112 --- /dev/null +++ b/python/testData/refactoring/extractmethod/AsyncDef.before.py @@ -0,0 +1,3 @@ +async def foo(x): + y = await x + return await y diff --git a/python/testData/refactoring/extractmethod/AwaitExpression.after.py b/python/testData/refactoring/extractmethod/AwaitExpression.after.py new file mode 100644 index 000000000000..1fdadf73f1b9 --- /dev/null +++ b/python/testData/refactoring/extractmethod/AwaitExpression.after.py @@ -0,0 +1,7 @@ +async def foo(x): + y = await bar(x) + return y + + +async def bar(x_new): + return await x_new + 1 diff --git a/python/testData/refactoring/extractmethod/AwaitExpression.before.py b/python/testData/refactoring/extractmethod/AwaitExpression.before.py new file mode 100644 index 000000000000..7b8c39b8c44a --- /dev/null +++ b/python/testData/refactoring/extractmethod/AwaitExpression.before.py @@ -0,0 +1,3 @@ +async def foo(x): + y = await x + 1 + return y diff --git a/python/testSrc/com/jetbrains/python/refactoring/PyExtractMethodTest.java b/python/testSrc/com/jetbrains/python/refactoring/PyExtractMethodTest.java index 1ebebf000efe..23e9f78bd958 100644 --- a/python/testSrc/com/jetbrains/python/refactoring/PyExtractMethodTest.java +++ b/python/testSrc/com/jetbrains/python/refactoring/PyExtractMethodTest.java @@ -278,4 +278,12 @@ public class PyExtractMethodTest extends LightMarkedTestCase { public void testProhibitedAtClassLevel() { doFail("foo", "Cannot perform refactoring at class level"); } + + public void testAsyncDef() { + doTest("bar", LanguageLevel.PYTHON35); + } + + public void testAwaitExpression() { + doTest("bar", LanguageLevel.PYTHON35); + } } From 2c118931123454503c6fe2f508ad8cd8f3b9fdfd Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 14 Sep 2015 11:51:36 +0300 Subject: [PATCH 15/15] Code completion and highlighting for special methods related to async-await (PY-16094) --- python/psi-api/src/com/jetbrains/python/PyNames.java | 8 +++++++- python/src/com/jetbrains/python/spellchecker/python.dic | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/psi-api/src/com/jetbrains/python/PyNames.java b/python/psi-api/src/com/jetbrains/python/PyNames.java index 3d77078f7982..9a93ac5d84b6 100644 --- a/python/psi-api/src/com/jetbrains/python/PyNames.java +++ b/python/psi-api/src/com/jetbrains/python/PyNames.java @@ -266,6 +266,7 @@ public class PyNames { private static final BuiltinDescription _self_other_descr = new BuiltinDescription("(self, other)"); private static final BuiltinDescription _self_item_descr = new BuiltinDescription("(self, item)"); private static final BuiltinDescription _self_key_descr = new BuiltinDescription("(self, key)"); + private static final BuiltinDescription _exit_descr = new BuiltinDescription("(self, exc_type, exc_val, exc_tb)"); private static final ImmutableMap BuiltinMethods = ImmutableMap.builder() .put("__abs__", _only_self_descr) @@ -291,7 +292,7 @@ public class PyNames { //_BuiltinMethods.put("__doc__", _only_self_descr); //_BuiltinMethods.put("__docformat__", _only_self_descr); .put("__enter__", _only_self_descr) - .put("__exit__", new BuiltinDescription("(self, exc_type, exc_val, exc_tb)")) + .put("__exit__", _exit_descr) .put("__eq__", _self_other_descr) //_BuiltinMethods.put("__file__", _only_self_descr); .put("__float__", _only_self_descr) @@ -397,6 +398,11 @@ public class PyNames { .put("__imatmul__", _self_other_descr) .put("__matmul__", _self_other_descr) .put("__rmatmul__", _self_other_descr) + .put("__await__", _only_self_descr) + .put("__aenter__", _only_self_descr) + .put("__aexit__", _exit_descr) + .put("__aiter__", _only_self_descr) + .put("__anext__", _only_self_descr) .build(); public static ImmutableMap getBuiltinMethods(LanguageLevel level) { diff --git a/python/src/com/jetbrains/python/spellchecker/python.dic b/python/src/com/jetbrains/python/spellchecker/python.dic index d5d09e51ebdb..de9d358620a7 100644 --- a/python/src/com/jetbrains/python/spellchecker/python.dic +++ b/python/src/com/jetbrains/python/spellchecker/python.dic @@ -32,10 +32,13 @@ addtoken addusersitepackages addval adpcm +aenter +aexit afterfork aggr aifc aiff +aiter alaw algor aliasmbcs @@ -46,6 +49,7 @@ altsep amost amper amperequal +anext anyobject apilevel appbundle