diff --git a/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java b/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java index 53e0de51af83..093b03a3b972 100644 --- a/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java +++ b/python/ipnb/src/org/jetbrains/plugins/ipnb/psi/IpnbPyParser.java @@ -32,7 +32,7 @@ public class IpnbPyParser extends PyParser { builder.setTokenTypeRemapper(statementParser); while (!builder.eof()) { - statementParser.parseStatement(context.emptyParsingScope()); + statementParser.parseStatement(); } rootMarker.done(root); return builder.getTreeBuilt(); diff --git a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java index 5d3060e155a6..be858df92181 100644 --- a/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java +++ b/python/psi-api/src/com/jetbrains/python/PyTokenTypes.java @@ -73,6 +73,7 @@ public class PyTokenTypes { public static final PyElementType FALSE_KEYWORD = new PyElementType("FALSE_KEYWORD"); public static final PyElementType NONLOCAL_KEYWORD = new PyElementType("NONLOCAL_KEYWORD"); public static final PyElementType DEBUG_KEYWORD = new PyElementType("DEBUG_KEYWORD"); + public static final PyElementType ASYNC_KEYWORD = new PyElementType("ASYNC_KEYWORD"); public static final PyElementType INTEGER_LITERAL = new PyElementType("INTEGER_LITERAL"); public static final PyElementType FLOAT_LITERAL = new PyElementType("FLOAT_LITERAL"); diff --git a/python/src/META-INF/python-core.xml b/python/src/META-INF/python-core.xml index 29489c86a618..3ed392a62d48 100644 --- a/python/src/META-INF/python-core.xml +++ b/python/src/META-INF/python-core.xml @@ -41,6 +41,7 @@ + @@ -614,6 +615,7 @@ + diff --git a/python/src/com/jetbrains/python/PythonTokenSetContributor.java b/python/src/com/jetbrains/python/PythonTokenSetContributor.java index 891eff28c507..51a42589109b 100644 --- a/python/src/com/jetbrains/python/PythonTokenSetContributor.java +++ b/python/src/com/jetbrains/python/PythonTokenSetContributor.java @@ -73,7 +73,7 @@ public class PythonTokenSetContributor extends PythonDialectsTokenSetContributor LAMBDA_KEYWORD, NOT_KEYWORD, OR_KEYWORD, PASS_KEYWORD, PRINT_KEYWORD, RAISE_KEYWORD, RETURN_KEYWORD, TRY_KEYWORD, WITH_KEYWORD, WHILE_KEYWORD, YIELD_KEYWORD, - NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD); + NONE_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, NONLOCAL_KEYWORD, DEBUG_KEYWORD, ASYNC_KEYWORD); } @NotNull diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java index 100330608c73..2d71ee03a685 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParser.java @@ -56,7 +56,7 @@ public class PyConsoleParser extends PyParser{ builder.setTokenTypeRemapper(stmt_parser); // must be done before touching the caching lexer with eof() call. while (!builder.eof()) { - stmt_parser.parseStatement(context.emptyParsingScope()); + stmt_parser.parseStatement(); } rootMarker.done(root); return builder.getTreeBuilt(); diff --git a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java index 32e8882b38b7..8c6aa7e33b8c 100644 --- a/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java +++ b/python/src/com/jetbrains/python/console/parsing/PyConsoleParsingContext.java @@ -72,7 +72,7 @@ public class PyConsoleParsingContext extends ParsingContext { @Override - public void parseStatement(ParsingScope scope) { + public void parseStatement() { if (myStartsWithIPythonSymbol) { parseIPythonCommand(); } @@ -89,7 +89,7 @@ public class PyConsoleParsingContext extends ParsingContext { myBuilder.advanceLexer(); } } - super.parseStatement(scope); + super.parseStatement(); } } @@ -101,13 +101,14 @@ public class PyConsoleParsingContext extends ParsingContext { ipythonCommand.done(PyElementTypes.EMPTY_EXPRESSION); } - protected void checkEndOfStatement(ParsingScope scope) { + protected void checkEndOfStatement() { if (myPythonConsoleData.isIPythonEnabled()) { PsiBuilder builder = myContext.getBuilder(); if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { builder.advanceLexer(); } else if (builder.getTokenType() == PyTokenTypes.SEMICOLON) { + final ParsingScope scope = getParsingContext().getScope(); if (!scope.isSuite()) { builder.advanceLexer(); if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { @@ -131,7 +132,7 @@ public class PyConsoleParsingContext extends ParsingContext { } } else { - super.checkEndOfStatement(scope); + super.checkEndOfStatement(); } } } diff --git a/python/src/com/jetbrains/python/parsing/FunctionParsing.java b/python/src/com/jetbrains/python/parsing/FunctionParsing.java index 60b32e91292c..2c8b7dff584b 100644 --- a/python/src/com/jetbrains/python/parsing/FunctionParsing.java +++ b/python/src/com/jetbrains/python/parsing/FunctionParsing.java @@ -20,6 +20,7 @@ import com.intellij.lang.WhitespacesBinders; import com.intellij.psi.tree.IElementType; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import org.jetbrains.annotations.NotNull; import static com.jetbrains.python.PyBundle.message; @@ -33,23 +34,25 @@ public class FunctionParsing extends Parsing { super(context); } - public void parseFunctionDeclaration() { + public void parseFunctionDeclaration(@NotNull PsiBuilder.Marker endMarker) { assertCurrentToken(PyTokenTypes.DEF_KEYWORD); - final PsiBuilder.Marker functionMarker = myBuilder.mark(); - parseFunctionInnards(functionMarker); + parseFunctionInnards(endMarker); } protected IElementType getFunctionType() { return FUNCTION_TYPE; } - protected void parseFunctionInnards(PsiBuilder.Marker functionMarker) { + protected void parseFunctionInnards(@NotNull PsiBuilder.Marker functionMarker) { myBuilder.advanceLexer(); parseIdentifierOrSkip(PyTokenTypes.LPAR); parseParameterList(); parseReturnTypeAnnotation(); checkMatches(PyTokenTypes.COLON, message("PARSE.expected.colon")); - getStatementParser().parseSuite(functionMarker, getFunctionType(), myContext.emptyParsingScope().withFunction(true)); + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withFunction(true)); + getStatementParser().parseSuite(functionMarker, getFunctionType()); + context.popScope(); } public void parseReturnTypeAnnotation() { @@ -63,7 +66,7 @@ public class FunctionParsing extends Parsing { } } - public void parseDecoratedDeclaration(ParsingScope scope) { + public void parseDecoratedDeclaration() { assertCurrentToken(PyTokenTypes.AT); // ??? need this? final PsiBuilder.Marker decoratorStartMarker = myBuilder.mark(); final PsiBuilder.Marker decoListMarker = myBuilder.mark(); @@ -92,15 +95,15 @@ public class FunctionParsing extends Parsing { } if (decorated) decoListMarker.done(PyElementTypes.DECORATOR_LIST); //else decoListMarker.rollbackTo(); - parseDeclarationAfterDecorator(decoratorStartMarker, scope); + parseDeclarationAfterDecorator(decoratorStartMarker); } - protected void parseDeclarationAfterDecorator(PsiBuilder.Marker endMarker, ParsingScope scope) { + protected void parseDeclarationAfterDecorator(PsiBuilder.Marker endMarker) { if (myBuilder.getTokenType() == PyTokenTypes.DEF_KEYWORD) { - parseFunctionInnards(endMarker); // it calls endMarker.done() + parseFunctionInnards(endMarker); } else if (myBuilder.getTokenType() == PyTokenTypes.CLASS_KEYWORD) { - getStatementParser().parseClassDeclaration(endMarker, scope); + getStatementParser().parseClassDeclaration(endMarker); } else { myBuilder.error(message("PARSE.expected.@.or.def")); diff --git a/python/src/com/jetbrains/python/parsing/ParsingContext.java b/python/src/com/jetbrains/python/parsing/ParsingContext.java index 66f3ebb32e62..94253c07720a 100644 --- a/python/src/com/jetbrains/python/parsing/ParsingContext.java +++ b/python/src/com/jetbrains/python/parsing/ParsingContext.java @@ -17,6 +17,10 @@ package com.jetbrains.python.parsing; import com.intellij.lang.PsiBuilder; import com.jetbrains.python.psi.LanguageLevel; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayDeque; +import java.util.Deque; public class ParsingContext { private final StatementParsing stmtParser; @@ -24,6 +28,7 @@ public class ParsingContext { private final FunctionParsing functionParser; private final PsiBuilder myBuilder; private final LanguageLevel myLanguageLevel; + private final Deque myScopes; public ParsingContext(final PsiBuilder builder, LanguageLevel languageLevel, StatementParsing.FUTURE futureFlag) { myBuilder = builder; @@ -31,6 +36,22 @@ public class ParsingContext { stmtParser = new StatementParsing(this, futureFlag); expressionParser = new ExpressionParsing(this); functionParser = new FunctionParsing(this); + myScopes = new ArrayDeque(); + myScopes.push(emptyParsingScope()); + } + + @NotNull + public ParsingScope popScope() { + return myScopes.pop(); + } + + public void pushScope(@NotNull ParsingScope scope) { + myScopes.push(scope); + } + + @NotNull + public ParsingScope getScope() { + return myScopes.peek(); } public StatementParsing getStatementParser() { diff --git a/python/src/com/jetbrains/python/parsing/ParsingScope.java b/python/src/com/jetbrains/python/parsing/ParsingScope.java index f029c2863a9a..d73721d52751 100644 --- a/python/src/com/jetbrains/python/parsing/ParsingScope.java +++ b/python/src/com/jetbrains/python/parsing/ParsingScope.java @@ -23,6 +23,9 @@ public class ParsingScope { private boolean myClass = false; private boolean mySuite = false; private boolean myAfterSemicolon = false; + private boolean myAsync = false; + + protected ParsingScope() {} public ParsingScope withFunction(boolean flag) { final ParsingScope result = copy(); @@ -42,6 +45,12 @@ public class ParsingScope { return result; } + public ParsingScope withAsync() { + final ParsingScope result = copy(); + result.myAsync = true; + return result; + } + public boolean isFunction() { return myFunction; } @@ -54,6 +63,10 @@ public class ParsingScope { return mySuite; } + public boolean isAsync() { + return myAsync; + } + public boolean isAfterSemicolon() { return myAfterSemicolon; } @@ -71,6 +84,7 @@ public class ParsingScope { result.myFunction = myFunction; result.myClass = myClass; result.mySuite = mySuite; + result.myAsync = myAsync; return result; } } diff --git a/python/src/com/jetbrains/python/parsing/PyParser.java b/python/src/com/jetbrains/python/parsing/PyParser.java index d7e431af46e3..631485670de8 100644 --- a/python/src/com/jetbrains/python/parsing/PyParser.java +++ b/python/src/com/jetbrains/python/parsing/PyParser.java @@ -49,14 +49,15 @@ public class PyParser implements PsiParser { builder.setTokenTypeRemapper(statementParser); // must be done before touching the caching lexer with eof() call. boolean lastAfterSemicolon = false; while (!builder.eof()) { - ParsingScope scope = context.emptyParsingScope(); + context.pushScope(context.emptyParsingScope()); if (lastAfterSemicolon) { - statementParser.parseSimpleStatement(scope); + statementParser.parseSimpleStatement(); } else { - statementParser.parseStatement(scope); + statementParser.parseStatement(); } - lastAfterSemicolon = scope.isAfterSemicolon(); + lastAfterSemicolon = context.getScope().isAfterSemicolon(); + context.popScope(); } rootMarker.done(root); ASTNode ast = builder.getTreeBuilt(); diff --git a/python/src/com/jetbrains/python/parsing/StatementParsing.java b/python/src/com/jetbrains/python/parsing/StatementParsing.java index 8de31cd09f27..bd6b423c033e 100644 --- a/python/src/com/jetbrains/python/parsing/StatementParsing.java +++ b/python/src/com/jetbrains/python/parsing/StatementParsing.java @@ -23,6 +23,7 @@ import com.intellij.psi.tree.TokenSet; import com.intellij.util.text.CharArrayUtil; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.psi.PyElementType; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.Nullable; @@ -49,6 +50,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { @NonNls protected static final String TOK_FALSE = "False"; @NonNls protected static final String TOK_NONLOCAL = "nonlocal"; @NonNls protected static final String TOK_EXEC = "exec"; + @NonNls protected static final String TOK_ASYNC = "async"; private static final String EXPRESSION_EXPECTED = "Expression expected"; public static final String IDENTIFIER_EXPECTED = "Identifier expected"; @@ -86,7 +88,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myBuilder.setTokenTypeRemapper(this); // clear cached token type } - public void parseStatement(ParsingScope scope) { + public void parseStatement() { while (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { myBuilder.advanceLexer(); @@ -98,103 +100,107 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (firstToken == null) return; if (firstToken == PyTokenTypes.WHILE_KEYWORD) { - parseWhileStatement(scope); + parseWhileStatement(); return; } if (firstToken == PyTokenTypes.IF_KEYWORD) { - parseIfStatement(PyTokenTypes.IF_KEYWORD, PyTokenTypes.ELIF_KEYWORD, PyTokenTypes.ELSE_KEYWORD, PyElementTypes.IF_STATEMENT, scope); + parseIfStatement(PyTokenTypes.IF_KEYWORD, PyTokenTypes.ELIF_KEYWORD, PyTokenTypes.ELSE_KEYWORD, PyElementTypes.IF_STATEMENT); return; } if (firstToken == PyTokenTypes.FOR_KEYWORD) { - parseForStatement(scope); + parseForStatement(myBuilder.mark()); return; } if (firstToken == PyTokenTypes.TRY_KEYWORD) { - parseTryStatement(scope); + parseTryStatement(); return; } if (firstToken == PyTokenTypes.DEF_KEYWORD) { - getFunctionParser().parseFunctionDeclaration(); + getFunctionParser().parseFunctionDeclaration(myBuilder.mark()); return; } if (firstToken == PyTokenTypes.AT) { - getFunctionParser().parseDecoratedDeclaration(scope); + getFunctionParser().parseDecoratedDeclaration(); return; } if (firstToken == PyTokenTypes.CLASS_KEYWORD) { - parseClassDeclaration(scope); + parseClassDeclaration(); return; } if (firstToken == PyTokenTypes.WITH_KEYWORD) { - parseWithStatement(scope); + parseWithStatement(myBuilder.mark()); + return; + } + if (firstToken == PyTokenTypes.ASYNC_KEYWORD) { + parseAsyncStatement(); return; } - parseSimpleStatement(scope); + parseSimpleStatement(); } - protected void parseSimpleStatement(ParsingScope scope) { + protected void parseSimpleStatement() { PsiBuilder builder = myContext.getBuilder(); final IElementType firstToken = builder.getTokenType(); if (firstToken == null) { return; } if (firstToken == PyTokenTypes.PRINT_KEYWORD && hasPrintStatement()) { - parsePrintStatement(builder, scope); + parsePrintStatement(builder); return; } if (firstToken == PyTokenTypes.ASSERT_KEYWORD) { - parseAssertStatement(scope); + parseAssertStatement(); return; } if (firstToken == PyTokenTypes.BREAK_KEYWORD) { - parseKeywordStatement(builder, PyElementTypes.BREAK_STATEMENT, scope); + parseKeywordStatement(builder, PyElementTypes.BREAK_STATEMENT); return; } if (firstToken == PyTokenTypes.CONTINUE_KEYWORD) { - parseKeywordStatement(builder, PyElementTypes.CONTINUE_STATEMENT, scope); + parseKeywordStatement(builder, PyElementTypes.CONTINUE_STATEMENT); return; } if (firstToken == PyTokenTypes.DEL_KEYWORD) { - parseDelStatement(scope); + parseDelStatement(); return; } if (firstToken == PyTokenTypes.EXEC_KEYWORD) { - parseExecStatement(scope); + parseExecStatement(); return; } if (firstToken == PyTokenTypes.GLOBAL_KEYWORD) { - parseNameDefiningStatement(scope, PyElementTypes.GLOBAL_STATEMENT); + parseNameDefiningStatement(PyElementTypes.GLOBAL_STATEMENT); return; } if (firstToken == PyTokenTypes.NONLOCAL_KEYWORD) { - parseNameDefiningStatement(scope, PyElementTypes.NONLOCAL_STATEMENT); + parseNameDefiningStatement(PyElementTypes.NONLOCAL_STATEMENT); return; } if (firstToken == PyTokenTypes.IMPORT_KEYWORD) { - parseImportStatement(scope, PyElementTypes.IMPORT_STATEMENT, PyElementTypes.IMPORT_ELEMENT); + parseImportStatement(PyElementTypes.IMPORT_STATEMENT, PyElementTypes.IMPORT_ELEMENT); return; } if (firstToken == PyTokenTypes.FROM_KEYWORD) { - parseFromImportStatement(scope); + parseFromImportStatement(); return; } if (firstToken == PyTokenTypes.PASS_KEYWORD) { - parseKeywordStatement(builder, PyElementTypes.PASS_STATEMENT, scope); + parseKeywordStatement(builder, PyElementTypes.PASS_STATEMENT); return; } if (firstToken == PyTokenTypes.RETURN_KEYWORD) { - parseReturnStatement(builder, scope); + parseReturnStatement(builder); return; } if (firstToken == PyTokenTypes.RAISE_KEYWORD) { - parseRaiseStatement(scope); + parseRaiseStatement(); return; } PsiBuilder.Marker exprStatement = builder.mark(); if (builder.getTokenType() == PyTokenTypes.YIELD_KEYWORD) { getExpressionParser().parseYieldOrTupleExpression(false); - checkEndOfStatement(scope); + checkEndOfStatement(); exprStatement.done(PyElementTypes.EXPRESSION_STATEMENT); return; } @@ -242,7 +248,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - checkEndOfStatement(scope); + checkEndOfStatement(); exprStatement.done(statementType); return; } @@ -270,8 +276,9 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { return myContext.getLanguageLevel().hasPrintStatement() && !myFutureFlags.contains(FUTURE.PRINT_FUNCTION); } - protected void checkEndOfStatement(ParsingScope scope) { + protected void checkEndOfStatement() { PsiBuilder builder = myContext.getBuilder(); + final ParsingScope scope = getParsingContext().getScope(); if (builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { builder.advanceLexer(); scope.setAfterSemicolon(false); @@ -291,7 +298,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - private void parsePrintStatement(final PsiBuilder builder, ParsingScope scope) { + private void parsePrintStatement(final PsiBuilder builder) { LOG.assertTrue(builder.getTokenType() == PyTokenTypes.PRINT_KEYWORD); final PsiBuilder.Marker statement = builder.mark(); builder.advanceLexer(); @@ -311,29 +318,29 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } getExpressionParser().parseSingleExpression(false); } - checkEndOfStatement(scope); + checkEndOfStatement(); statement.done(PyElementTypes.PRINT_STATEMENT); } - protected void parseKeywordStatement(PsiBuilder builder, IElementType statementType, ParsingScope scope) { + protected void parseKeywordStatement(PsiBuilder builder, IElementType statementType) { final PsiBuilder.Marker statement = builder.mark(); builder.advanceLexer(); - checkEndOfStatement(scope); + checkEndOfStatement(); statement.done(statementType); } - private void parseReturnStatement(PsiBuilder builder, ParsingScope inSuite) { + private void parseReturnStatement(PsiBuilder builder) { LOG.assertTrue(builder.getTokenType() == PyTokenTypes.RETURN_KEYWORD); final PsiBuilder.Marker returnStatement = builder.mark(); builder.advanceLexer(); if (builder.getTokenType() != null && !getEndOfStatementsTokens().contains(builder.getTokenType())) { getExpressionParser().parseExpression(); } - checkEndOfStatement(inSuite); + checkEndOfStatement(); returnStatement.done(PyElementTypes.RETURN_STATEMENT); } - private void parseDelStatement(ParsingScope inSuite) { + private void parseDelStatement() { assertCurrentToken(PyTokenTypes.DEL_KEYWORD); final PsiBuilder.Marker delStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -349,11 +356,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - checkEndOfStatement(inSuite); + checkEndOfStatement(); delStatement.done(PyElementTypes.DEL_STATEMENT); } - private void parseRaiseStatement(ParsingScope inSuite) { + private void parseRaiseStatement() { assertCurrentToken(PyTokenTypes.RAISE_KEYWORD); final PsiBuilder.Marker raiseStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -374,11 +381,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } } - checkEndOfStatement(inSuite); + checkEndOfStatement(); raiseStatement.done(PyElementTypes.RAISE_STATEMENT); } - private void parseAssertStatement(ParsingScope scope) { + private void parseAssertStatement() { assertCurrentToken(PyTokenTypes.ASSERT_KEYWORD); final PsiBuilder.Marker assertStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -389,7 +396,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myContext.getBuilder().error(EXPRESSION_EXPECTED); } } - checkEndOfStatement(scope); + checkEndOfStatement(); } else { myContext.getBuilder().error(EXPRESSION_EXPECTED); @@ -397,12 +404,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { assertStatement.done(PyElementTypes.ASSERT_STATEMENT); } - protected void parseImportStatement(ParsingScope scope, IElementType statementType, IElementType elementType) { + protected void parseImportStatement(IElementType statementType, IElementType elementType) { final PsiBuilder builder = myContext.getBuilder(); final PsiBuilder.Marker importStatement = builder.mark(); builder.advanceLexer(); parseImportElements(elementType, true, false, false); - checkEndOfStatement(scope); + checkEndOfStatement(); importStatement.done(statementType); } @@ -411,7 +418,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { from identifier import id, id... -- may be either relative or absolute from . import identifier -- only relative */ - private void parseFromImportStatement(ParsingScope inSuite) { + private void parseFromImportStatement() { PsiBuilder builder = myContext.getBuilder(); assertCurrentToken(PyTokenTypes.FROM_KEYWORD); myFutureImportPhase = Phase.FROM; @@ -447,7 +454,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { statementType = types.statement; parseImportElements(types.element, false, false, from_future); } - checkEndOfStatement(inSuite); + checkEndOfStatement(); fromImportStatement.done(statementType); myFutureImportPhase = Phase.NONE; } @@ -563,7 +570,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { return true; } - private void parseNameDefiningStatement(ParsingScope scope, final PyElementType elementType) { + private void parseNameDefiningStatement(final PyElementType elementType) { final PsiBuilder.Marker globalStatement = myBuilder.mark(); myBuilder.advanceLexer(); parseIdentifier(PyElementTypes.TARGET_EXPRESSION); @@ -571,11 +578,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myBuilder.advanceLexer(); parseIdentifier(PyElementTypes.TARGET_EXPRESSION); } - checkEndOfStatement(scope); + checkEndOfStatement(); globalStatement.done(elementType); } - private void parseExecStatement(ParsingScope inSuite) { + private void parseExecStatement() { assertCurrentToken(PyTokenTypes.EXEC_KEYWORD); final PsiBuilder.Marker execStatement = myBuilder.mark(); myBuilder.advanceLexer(); @@ -588,12 +595,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { getExpressionParser().parseSingleExpression(false); } } - checkEndOfStatement(inSuite); + checkEndOfStatement(); execStatement.done(PyElementTypes.EXEC_STATEMENT); } - protected void parseIfStatement(PyElementType ifKeyword, PyElementType elifKeyword, PyElementType elseKeyword, PyElementType elementType, - ParsingScope scope) { + protected void parseIfStatement(PyElementType ifKeyword, PyElementType elifKeyword, PyElementType elseKeyword, + PyElementType elementType) { assertCurrentToken(ifKeyword); final PsiBuilder.Marker ifStatement = myBuilder.mark(); final PsiBuilder.Marker ifPart = myBuilder.mark(); @@ -601,7 +608,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (!getExpressionParser().parseSingleExpression(false)) { myBuilder.error("expression expected"); } - parseColonAndSuite(scope); + parseColonAndSuite(); ifPart.done(PyElementTypes.IF_PART_IF); PsiBuilder.Marker elifPart = myBuilder.mark(); while (myBuilder.getTokenType() == elifKeyword) { @@ -609,7 +616,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (!getExpressionParser().parseSingleExpression(false)) { myBuilder.error("expression expected"); } - parseColonAndSuite(scope); + parseColonAndSuite(); elifPart.done(PyElementTypes.IF_PART_ELIF); elifPart = myBuilder.mark(); } @@ -617,7 +624,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == elseKeyword) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { @@ -647,33 +654,32 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { return result; } - private void parseForStatement(ParsingScope scope) { + private void parseForStatement(PsiBuilder.Marker endMarker) { assertCurrentToken(PyTokenTypes.FOR_KEYWORD); - final PsiBuilder.Marker statement = myBuilder.mark(); - parseForPart(scope); + parseForPart(); final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { elsePart.drop(); } - statement.done(PyElementTypes.FOR_STATEMENT); + endMarker.done(PyElementTypes.FOR_STATEMENT); } - protected void parseForPart(ParsingScope scope) { + protected void parseForPart() { final PsiBuilder.Marker forPart = myBuilder.mark(); myBuilder.advanceLexer(); getExpressionParser().parseExpression(true, true); checkMatches(PyTokenTypes.IN_KEYWORD, "'in' expected"); getExpressionParser().parseExpression(); - parseColonAndSuite(scope); + parseColonAndSuite(); forPart.done(PyElementTypes.FOR_PART); } - private void parseWhileStatement(ParsingScope scope) { + private void parseWhileStatement() { assertCurrentToken(PyTokenTypes.WHILE_KEYWORD); final PsiBuilder.Marker statement = myBuilder.mark(); final PsiBuilder.Marker whilePart = myBuilder.mark(); @@ -681,12 +687,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (!getExpressionParser().parseSingleExpression(false)) { myBuilder.error(EXPRESSION_EXPECTED); } - parseColonAndSuite(scope); + parseColonAndSuite(); whilePart.done(PyElementTypes.WHILE_PART); final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { @@ -695,12 +701,12 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { statement.done(PyElementTypes.WHILE_STATEMENT); } - private void parseTryStatement(ParsingScope scope) { + private void parseTryStatement() { assertCurrentToken(PyTokenTypes.TRY_KEYWORD); final PsiBuilder.Marker statement = myBuilder.mark(); final PsiBuilder.Marker tryPart = myBuilder.mark(); myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); tryPart.done(PyElementTypes.TRY_PART); boolean haveExceptClause = false; if (myBuilder.getTokenType() == PyTokenTypes.EXCEPT_KEYWORD) { @@ -720,13 +726,13 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } } - parseColonAndSuite(scope); + parseColonAndSuite(); exceptBlock.done(PyElementTypes.EXCEPT_PART); } final PsiBuilder.Marker elsePart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); elsePart.done(PyElementTypes.ELSE_PART); } else { @@ -736,7 +742,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker finallyPart = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.FINALLY_KEYWORD) { myBuilder.advanceLexer(); - parseColonAndSuite(scope); + parseColonAndSuite(); finallyPart.done(PyElementTypes.FINALLY_PART); } else { @@ -750,9 +756,9 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { statement.done(PyElementTypes.TRY_EXCEPT_STATEMENT); } - private void parseColonAndSuite(ParsingScope scope) { + private void parseColonAndSuite() { if (expectColon()) { - parseSuite(scope); + parseSuite(); } else { final PsiBuilder.Marker mark = myBuilder.mark(); @@ -760,9 +766,8 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } } - private void parseWithStatement(ParsingScope scope) { + private void parseWithStatement(PsiBuilder.Marker endMarker) { assertCurrentToken(PyTokenTypes.WITH_KEYWORD); - final PsiBuilder.Marker statement = myBuilder.mark(); myBuilder.advanceLexer(); while (true) { PsiBuilder.Marker withItem = myBuilder.mark(); @@ -780,16 +785,16 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { break; } } - parseColonAndSuite(scope); - statement.done(PyElementTypes.WITH_STATEMENT); + parseColonAndSuite(); + endMarker.done(PyElementTypes.WITH_STATEMENT); } - private void parseClassDeclaration(ParsingScope scope) { + private void parseClassDeclaration() { final PsiBuilder.Marker classMarker = myBuilder.mark(); - parseClassDeclaration(classMarker, scope); + parseClassDeclaration(classMarker); } - public void parseClassDeclaration(PsiBuilder.Marker classMarker, ParsingScope scope) { + public void parseClassDeclaration(PsiBuilder.Marker classMarker) { assertCurrentToken(PyTokenTypes.CLASS_KEYWORD); myBuilder.advanceLexer(); parseIdentifierOrSkip(PyTokenTypes.LPAR, PyTokenTypes.COLON); @@ -800,15 +805,41 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { final PsiBuilder.Marker inheritMarker = myBuilder.mark(); inheritMarker.done(PyElementTypes.ARGUMENT_LIST); } - parseColonAndSuite(scope.withClass(true)); + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withClass(true)); + parseColonAndSuite(); + context.popScope(); classMarker.done(PyElementTypes.CLASS_DECLARATION); } - public void parseSuite(ParsingScope scope) { - parseSuite(null, null, scope); + private void parseAsyncStatement() { + assertCurrentToken(PyTokenTypes.ASYNC_KEYWORD); + final PsiBuilder.Marker marker = myBuilder.mark(); + myBuilder.advanceLexer(); + final IElementType token = myBuilder.getTokenType(); + if (token == PyTokenTypes.DEF_KEYWORD) { + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withAsync()); + getFunctionParser().parseFunctionDeclaration(marker); + context.popScope(); + } + else if (token == PyTokenTypes.WITH_KEYWORD) { + parseWithStatement(marker); + } + else if (token == PyTokenTypes.FOR_KEYWORD) { + parseForStatement(marker); + } + else { + marker.drop(); + myBuilder.error("'def' or 'with' or 'for' expected"); + } } - public void parseSuite(@Nullable PsiBuilder.Marker endMarker, @Nullable IElementType elType, ParsingScope scope) { + public void parseSuite() { + parseSuite(null, null); + } + + public void parseSuite(@Nullable PsiBuilder.Marker endMarker, @Nullable IElementType elType) { if (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { myBuilder.advanceLexer(); @@ -821,7 +852,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } else { while (!myBuilder.eof() && myBuilder.getTokenType() != PyTokenTypes.DEDENT) { - parseStatement(scope); + parseStatement(); } } } @@ -837,10 +868,6 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (indentFound && !myBuilder.eof()) { checkMatches(PyTokenTypes.DEDENT, "Dedent expected"); } - // NOTE: the following line advances the PsiBuilder lexer and thus - // ensures that the whitespace following the statement list is included - // in the block containing the statement list - myBuilder.getTokenType(); } else { final PsiBuilder.Marker marker = myBuilder.mark(); @@ -848,12 +875,17 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { myBuilder.error("Statement expected"); } else { - parseSimpleStatement(scope.withSuite(true)); + final ParsingContext context = getParsingContext(); + context.pushScope(context.getScope().withSuite(true)); + parseSimpleStatement(); + context.popScope(); while (matchToken(PyTokenTypes.SEMICOLON)) { if (matchToken(PyTokenTypes.STATEMENT_BREAK)) { break; } - parseSimpleStatement(scope.withSuite(true)); + context.pushScope(context.getScope().withSuite(true)); + parseSimpleStatement(); + context.popScope(); } } marker.done(PyElementTypes.STATEMENT_LIST); @@ -905,6 +937,11 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { if (isWordAtPosition(text, start, end, TOK_NONLOCAL)) { return PyTokenTypes.NONLOCAL_KEYWORD; } + if (myContext.getLanguageLevel().isAtLeast(LanguageLevel.PYTHON35) && isWordAtPosition(text, start, end, TOK_ASYNC)) { + if (myContext.getScope().isAsync() || myBuilder.lookAhead(1) == PyTokenTypes.DEF_KEYWORD) { + return PyTokenTypes.ASYNC_KEYWORD; + } + } } else if (!myContext.getLanguageLevel().isPy3K() && source == PyTokenTypes.IDENTIFIER) { if (isWordAtPosition(text, start, end, TOK_EXEC)) { diff --git a/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java new file mode 100644 index 000000000000..717d59f8e37d --- /dev/null +++ b/python/src/com/jetbrains/python/validation/DumbAwareHighlightingAnnotator.java @@ -0,0 +1,62 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.validation; + +import com.intellij.codeInsight.daemon.impl.HighlightRangeExtension; +import com.intellij.lang.ASTNode; +import com.intellij.lang.annotation.Annotation; +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiFile; +import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.highlighting.PyHighlighter; +import com.jetbrains.python.psi.PyFile; +import com.jetbrains.python.psi.PyForStatement; +import com.jetbrains.python.psi.PyFunction; +import com.jetbrains.python.psi.PyWithStatement; +import org.jetbrains.annotations.NotNull; + +/** + * @author vlan + */ +public class DumbAwareHighlightingAnnotator extends PyAnnotator implements HighlightRangeExtension { + @Override + public void visitPyFunction(PyFunction node) { + highlightAsyncKeyword(node); + } + + @Override + public void visitPyForStatement(PyForStatement node) { + highlightAsyncKeyword(node); + } + + @Override + public void visitPyWithStatement(PyWithStatement node) { + highlightAsyncKeyword(node); + } + + @Override + public boolean isForceHighlightParents(@NotNull PsiFile file) { + return file instanceof PyFile; + } + + private void highlightAsyncKeyword(@NotNull PsiElement node) { + final ASTNode asyncNode = node.getNode().findChildByType(PyTokenTypes.ASYNC_KEYWORD); + if (asyncNode != null) { + final Annotation annotation = getHolder().createInfoAnnotation(asyncNode, null); + annotation.setTextAttributes(PyHighlighter.PY_KEYWORD); + } + } +} diff --git a/python/testData/highlighting/async.py b/python/testData/highlighting/async.py new file mode 100644 index 000000000000..545f5ebf2c91 --- /dev/null +++ b/python/testData/highlighting/async.py @@ -0,0 +1,28 @@ +async def foo(): + pass + +async = 1 + +async def bar(): + pass + + +async def # Incomplete + + +def regular(xs): + + async def quux(): + async for x in xs: + pass + + async with xs: + pass + + async for x in xs: + pass + + async with xs: + pass + + return async diff --git a/python/testData/highlighting/returnOutsideOfFunction.py b/python/testData/highlighting/returnOutsideOfFunction.py index 8ce3bb17b73d..78054559568a 100644 --- a/python/testData/highlighting/returnOutsideOfFunction.py +++ b/python/testData/highlighting/returnOutsideOfFunction.py @@ -1,3 +1,3 @@ -def foo(): - class C: +def foo(): + class C: return 1 \ No newline at end of file diff --git a/python/testData/highlighting/returnWithArgumentsInGenerator.py b/python/testData/highlighting/returnWithArgumentsInGenerator.py index f16d817498ed..4c5ba91e5aba 100644 --- a/python/testData/highlighting/returnWithArgumentsInGenerator.py +++ b/python/testData/highlighting/returnWithArgumentsInGenerator.py @@ -1,3 +1,3 @@ -def f(): +def f(): yield 42 return 28 diff --git a/python/testData/psi/AsyncDef.py b/python/testData/psi/AsyncDef.py new file mode 100644 index 000000000000..4a7b81c00582 --- /dev/null +++ b/python/testData/psi/AsyncDef.py @@ -0,0 +1,15 @@ +async def foo(x, y): + pass + + async def foo_nested(): + pass + + +async = 10 + + +def bar(): + print(async) + + async def bar_nested(): + pass diff --git a/python/testData/psi/AsyncDef.txt b/python/testData/psi/AsyncDef.txt new file mode 100644 index 000000000000..15792ae90d92 --- /dev/null +++ b/python/testData/psi/AsyncDef.txt @@ -0,0 +1,80 @@ +PyFile:AsyncDef.py + PyFunction('foo') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PyNamedParameter('x') + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyNamedParameter('y') + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n ') + PyFunction('foo_nested') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo_nested') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n\n') + PyAssignmentStatement + PyTargetExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('10') + PsiWhiteSpace('\n\n\n') + PyFunction('bar') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyExpressionStatement + PyCallExpression: print + PyReferenceExpression: print + PsiElement(Py:IDENTIFIER)('print') + PyArgumentList + PsiElement(Py:LPAR)('(') + PyReferenceExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiElement(Py:RPAR)(')') + PsiWhiteSpace('\n\n ') + PyFunction('bar_nested') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar_nested') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testData/psi/AsyncFor.py b/python/testData/psi/AsyncFor.py new file mode 100644 index 000000000000..2eb8f7abe936 --- /dev/null +++ b/python/testData/psi/AsyncFor.py @@ -0,0 +1,7 @@ +async def f(): + async for x in xs: + pass + + +async for y in ys: + pass diff --git a/python/testData/psi/AsyncFor.txt b/python/testData/psi/AsyncFor.txt new file mode 100644 index 000000000000..abadd01f0402 --- /dev/null +++ b/python/testData/psi/AsyncFor.txt @@ -0,0 +1,54 @@ +PyFile:AsyncFor.py + PyFunction('f') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('f') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyForStatement + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PyForPart + PsiElement(Py:FOR_KEYWORD)('for') + PsiWhiteSpace(' ') + PyTargetExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiWhiteSpace(' ') + PsiElement(Py:IN_KEYWORD)('in') + PsiWhiteSpace(' ') + PyReferenceExpression: xs + PsiElement(Py:IDENTIFIER)('xs') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n\n') + PyExpressionStatement + PyReferenceExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiErrorElement:End of statement expected + + PsiWhiteSpace(' ') + PyForStatement + PyForPart + PsiElement(Py:FOR_KEYWORD)('for') + PsiWhiteSpace(' ') + PyTargetExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiWhiteSpace(' ') + PsiElement(Py:IN_KEYWORD)('in') + PsiWhiteSpace(' ') + PyReferenceExpression: ys + PsiElement(Py:IDENTIFIER)('ys') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testData/psi/AsyncWith.py b/python/testData/psi/AsyncWith.py new file mode 100644 index 000000000000..6409856e4ce6 --- /dev/null +++ b/python/testData/psi/AsyncWith.py @@ -0,0 +1,6 @@ +async def foo(): + async with x: + pass + +async with y: + pass diff --git a/python/testData/psi/AsyncWith.txt b/python/testData/psi/AsyncWith.txt new file mode 100644 index 000000000000..0313005b086d --- /dev/null +++ b/python/testData/psi/AsyncWith.txt @@ -0,0 +1,44 @@ +PyFile:AsyncWith.py + PyFunction('foo') + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyWithStatement + PsiElement(Py:ASYNC_KEYWORD)('async') + PsiWhiteSpace(' ') + PsiElement(Py:WITH_KEYWORD)('with') + PsiWhiteSpace(' ') + PyWithItem + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n\n') + PyExpressionStatement + PyReferenceExpression: async + PsiElement(Py:IDENTIFIER)('async') + PsiErrorElement:End of statement expected + + PsiWhiteSpace(' ') + PyWithStatement + PsiElement(Py:WITH_KEYWORD)('with') + PsiWhiteSpace(' ') + PyWithItem + PyReferenceExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java index 73ab49e28659..83e625b7d3bc 100644 --- a/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonHighlightingTest.java @@ -229,6 +229,10 @@ public class PythonHighlightingTest extends PyTestCase { doTest(); } + public void testAsync() { + doTest(LanguageLevel.PYTHON35, true, true); + } + // --- private void doTest(final LanguageLevel languageLevel, final boolean checkWarnings, final boolean checkInfos) { PythonLanguageLevelPusher.setForcedLanguageLevel(myFixture.getProject(), languageLevel); diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 56a4c0523f64..ecf2c39f46f3 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -486,6 +486,18 @@ public class PythonParsingTest extends ParsingTestCase { doTest(); } + public void testAsyncDef() { + doTest(LanguageLevel.PYTHON35); + } + + public void testAsyncWith() { + doTest(LanguageLevel.PYTHON35); + } + + public void testAsyncFor() { + doTest(LanguageLevel.PYTHON35); + } + public void doTest(LanguageLevel languageLevel) { LanguageLevel prev = myLanguageLevel; myLanguageLevel = languageLevel;