diff --git a/python/src/com/jetbrains/python/parsing/StatementParsing.java b/python/src/com/jetbrains/python/parsing/StatementParsing.java index bcfb0a7ba1ce..be443b66402d 100644 --- a/python/src/com/jetbrains/python/parsing/StatementParsing.java +++ b/python/src/com/jetbrains/python/parsing/StatementParsing.java @@ -16,23 +16,42 @@ package com.jetbrains.python.parsing; +import com.intellij.lang.ITokenTypeRemapper; import com.intellij.lang.PsiBuilder; import com.intellij.openapi.diagnostic.Logger; import com.intellij.psi.tree.IElementType; +import com.intellij.util.text.CharArrayUtil; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; +import org.jetbrains.annotations.Nullable; + +import java.util.EnumSet; +import java.util.Set; + /** * @author yole */ -public class StatementParsing extends Parsing { +public class StatementParsing + extends Parsing + implements ITokenTypeRemapper +{ private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.parsing.StatementParsing"); + protected enum FIPH {NONE, FROM, FUTURE, IMPORT} // 'from __future__ import' phase + private FIPH _from_import_phase = FIPH.NONE; + private boolean _expect_AS_kwd = false; + + protected enum FUTURE {ABSOLUTE_IMPORT, DIVISION, GENERATORS, NESTED_SCOPES, WITH_STATEMENT} + protected Set myFutureFlags = EnumSet.noneOf(FUTURE.class); protected StatementParsing(ParsingContext context) { super(context); } public void parseStatement() { + // TODO: initialize it in a more proper place + myBuilder.setTokenTypeRemapper(this); + while (myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK) { myBuilder.advanceLexer(); } @@ -308,7 +327,7 @@ public class StatementParsing extends Parsing { LOG.assertTrue(builder.getTokenType() == PyTokenTypes.IMPORT_KEYWORD); final PsiBuilder.Marker importStatement = builder.mark(); builder.advanceLexer(); - parseImportElements(true, false); + parseImportElements(true, false, false); checkEndOfStatement(inSuite); importStatement.done(PyElementTypes.IMPORT_STATEMENT); } @@ -316,48 +335,66 @@ public class StatementParsing extends Parsing { private void parseFromImportStatement(boolean inSuite) { PsiBuilder builder = myContext.getBuilder(); assertCurrentToken(PyTokenTypes.FROM_KEYWORD); + _from_import_phase = FIPH.FROM; final PsiBuilder.Marker fromImportStatement = builder.mark(); builder.advanceLexer(); + boolean from_future = false; if (parseDottedName()) { checkMatches(PyTokenTypes.IMPORT_KEYWORD, "'import' expected"); + if (_from_import_phase == FIPH.FUTURE) { + _from_import_phase = FIPH.IMPORT; + from_future = true; + } if (builder.getTokenType() == PyTokenTypes.MULT) { builder.advanceLexer(); } else if (builder.getTokenType() == PyTokenTypes.LPAR) { builder.advanceLexer(); - parseImportElements(false, true); + parseImportElements(false, true, from_future); checkMatches(PyTokenTypes.RPAR, ") expected"); } else { - parseImportElements(false, false); + parseImportElements(false, false, from_future); } } checkEndOfStatement(inSuite); fromImportStatement.done(PyElementTypes.FROM_IMPORT_STATEMENT); + _from_import_phase = FIPH.NONE; } - private void parseImportElements(boolean isModuleName, boolean inParens) { + private void parseImportElements(boolean is_module_import, boolean in_parens, final boolean from_future) { PsiBuilder builder = myContext.getBuilder(); while (true) { final PsiBuilder.Marker asMarker = builder.mark(); - if (isModuleName) { - if (!parseDottedName()) { - asMarker.drop(); + if (is_module_import) { // import _ + if (!parseDottedNameAsAware(true)) { + asMarker.drop(); break; } } - else { - parseIdentifier(PyElementTypes.REFERENCE_EXPRESSION); + else { // from X import _ + String token_text = parseIdentifier(PyElementTypes.REFERENCE_EXPRESSION); + if (from_future) { + // TODO: make constants for known future feature names + if ("with_statement".equals(token_text)) { + myFutureFlags.add(FUTURE.WITH_STATEMENT); + } + else if ("nested_scopes".equals(token_text)) { + myFutureFlags.add(FUTURE.NESTED_SCOPES); + } + } } - String tokenText = builder.getTokenText(); - if (builder.getTokenType() == PyTokenTypes.IDENTIFIER && tokenText != null && tokenText.equals("as")) { + _expect_AS_kwd = true; // possible 'as' comes as an ident; reparse it as keyword if found + if (builder.getTokenType() == PyTokenTypes.AS_KEYWORD) { builder.advanceLexer(); + _expect_AS_kwd = false; parseIdentifier(PyElementTypes.TARGET_EXPRESSION); } asMarker.done(PyElementTypes.IMPORT_ELEMENT); + _expect_AS_kwd = false; if (builder.getTokenType() == PyTokenTypes.COMMA) { builder.advanceLexer(); - if (inParens && builder.getTokenType() == PyTokenTypes.RPAR) { + if (in_parens && builder.getTokenType() == PyTokenTypes.RPAR) { break; } } @@ -367,19 +404,27 @@ public class StatementParsing extends Parsing { } } - private void parseIdentifier(final IElementType elementType) { + @Nullable + private String parseIdentifier(final IElementType elementType) { final PsiBuilder.Marker idMarker = myBuilder.mark(); if (myBuilder.getTokenType() == PyTokenTypes.IDENTIFIER) { + String id_text = myBuilder.getTokenText(); myBuilder.advanceLexer(); idMarker.done(elementType); + return id_text; } else { myBuilder.error("identifier expected"); idMarker.drop(); } + return null; } public boolean parseDottedName() { + return parseDottedNameAsAware(false); + } + + protected boolean parseDottedNameAsAware(boolean expect_as) { if (myBuilder.getTokenType() != PyTokenTypes.IDENTIFIER) { myBuilder.error("identifier expected"); return false; @@ -387,12 +432,15 @@ public class StatementParsing extends Parsing { PsiBuilder.Marker marker = myBuilder.mark(); myBuilder.advanceLexer(); marker.done(PyElementTypes.REFERENCE_EXPRESSION); + boolean old_expect_AS_kwd = _expect_AS_kwd; + _expect_AS_kwd = expect_as; while (myBuilder.getTokenType() == PyTokenTypes.DOT) { marker = marker.precede(); myBuilder.advanceLexer(); checkMatches(PyTokenTypes.IDENTIFIER, "identifier expected"); marker.done(PyElementTypes.REFERENCE_EXPRESSION); } + _expect_AS_kwd = old_expect_AS_kwd; return true; } @@ -582,7 +630,7 @@ public class StatementParsing extends Parsing { if (!myBuilder.eof()) { checkMatches(PyTokenTypes.DEDENT, "dedent expected"); } - // HACK: the following line advances the PsiBuilder lexer and thus + // NOTE: the following line advances the PsiBuilder lexer and thus // ensures that the whitespace following the statement list is included // in the block containing the statement list myBuilder.getTokenType(); @@ -600,4 +648,29 @@ public class StatementParsing extends Parsing { } } } + public IElementType filter(final IElementType source, final int start, final int end, final CharSequence text) { + if ( + (myFutureFlags.contains(FUTURE.WITH_STATEMENT) || _expect_AS_kwd) && + source == PyTokenTypes.IDENTIFIER && + CharArrayUtil.regionMatches(text, start, end, "as") + ) { + return PyTokenTypes.AS_KEYWORD; + } + else if ( // filter + (_from_import_phase == FIPH.FROM) && + source == PyTokenTypes.IDENTIFIER && + CharArrayUtil.regionMatches(text, start, end, "__future__") + ) { + _from_import_phase = FIPH.FUTURE; + return source; + } + else if ( + myFutureFlags.contains(FUTURE.WITH_STATEMENT) && + source == PyTokenTypes.IDENTIFIER && + CharArrayUtil.regionMatches(text, start, end, "with") + ) { + return PyTokenTypes.WITH_KEYWORD; + } + return source; + } } diff --git a/python/testData/psi/ImportStmt.py b/python/testData/psi/ImportStmt.py new file mode 100644 index 000000000000..d7dc0ba0968e --- /dev/null +++ b/python/testData/psi/ImportStmt.py @@ -0,0 +1,22 @@ +# import A as B +import A as B +# import A as D, A as DDD +import A as D, A as DDD +# import A.B.C as D, A as DDD +import A.B.C as D, A as DDD +# import as as as +import as as as +# import A +import A +# import A, B, C +import A, B, C +# import A.B.C +import A.B.C +# from A import B, C +from A import B, C +# from A import B as C +from A import B as C +# from A import B as C, X as Y +from A import B as C, X as Y +# from foo import as as as +from foo import as as as \ No newline at end of file diff --git a/python/testData/psi/ImportStmt.txt b/python/testData/psi/ImportStmt.txt new file mode 100644 index 000000000000..0a81bf175c4f --- /dev/null +++ b/python/testData/psi/ImportStmt.txt @@ -0,0 +1,211 @@ +PyFile:ImportStmt.py + PsiComment(Py:END_OF_LINE_COMMENT)('# import A as B') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: B + PsiElement(Py:IDENTIFIER)('B') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# import A as D, A as DDD') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: D + PsiElement(Py:IDENTIFIER)('D') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: DDD + PsiElement(Py:IDENTIFIER)('DDD') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# import A.B.C as D, A as DDD') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: C + PyReferenceExpression: B + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiElement(Py:DOT)('.') + PsiElement(Py:IDENTIFIER)('B') + PsiElement(Py:DOT)('.') + PsiElement(Py:IDENTIFIER)('C') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: D + PsiElement(Py:IDENTIFIER)('D') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: DDD + PsiElement(Py:IDENTIFIER)('DDD') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# import as as as') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: as + PsiElement(Py:IDENTIFIER)('as') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: as + PsiElement(Py:IDENTIFIER)('as') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# import A') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# import A, B, C') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: B + PsiElement(Py:IDENTIFIER)('B') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: C + PsiElement(Py:IDENTIFIER)('C') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# import A.B.C') + PsiWhiteSpace('\n') + PyImportStatement + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: C + PyReferenceExpression: B + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiElement(Py:DOT)('.') + PsiElement(Py:IDENTIFIER)('B') + PsiElement(Py:DOT)('.') + PsiElement(Py:IDENTIFIER)('C') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# from A import B, C') + PsiWhiteSpace('\n') + PyFromImportStatement + PsiElement(Py:FROM_KEYWORD)('from') + PsiWhiteSpace(' ') + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: B + PsiElement(Py:IDENTIFIER)('B') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: C + PsiElement(Py:IDENTIFIER)('C') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# from A import B as C') + PsiWhiteSpace('\n') + PyFromImportStatement + PsiElement(Py:FROM_KEYWORD)('from') + PsiWhiteSpace(' ') + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: B + PsiElement(Py:IDENTIFIER)('B') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: C + PsiElement(Py:IDENTIFIER)('C') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# from A import B as C, X as Y') + PsiWhiteSpace('\n') + PyFromImportStatement + PsiElement(Py:FROM_KEYWORD)('from') + PsiWhiteSpace(' ') + PyReferenceExpression: A + PsiElement(Py:IDENTIFIER)('A') + PsiWhiteSpace(' ') + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: B + PsiElement(Py:IDENTIFIER)('B') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: C + PsiElement(Py:IDENTIFIER)('C') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: X + PsiElement(Py:IDENTIFIER)('X') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: Y + PsiElement(Py:IDENTIFIER)('Y') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# from foo import as as as') + PsiWhiteSpace('\n') + PyFromImportStatement + PsiElement(Py:FROM_KEYWORD)('from') + PsiWhiteSpace(' ') + PyReferenceExpression: foo + PsiElement(Py:IDENTIFIER)('foo') + PsiWhiteSpace(' ') + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: as + PsiElement(Py:IDENTIFIER)('as') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyTargetExpression: as + PsiElement(Py:IDENTIFIER)('as') \ No newline at end of file diff --git a/python/testData/psi/WithStatement2.py b/python/testData/psi/WithStatement2.py new file mode 100644 index 000000000000..b2a91fa28608 --- /dev/null +++ b/python/testData/psi/WithStatement2.py @@ -0,0 +1,9 @@ +# with = 1 # legal identifier +# with = 1 +with = 1 +# from __future__ import with_statement, nested_scopes +from __future__ import with_statement, nested_scopes +# with x.y(z)[t] as y: pass +with x.y(z)[t] as y: pass +# with = 1 # now illegal +with = 1 diff --git a/python/testData/psi/WithStatement2.txt b/python/testData/psi/WithStatement2.txt new file mode 100644 index 000000000000..826c7fe96d15 --- /dev/null +++ b/python/testData/psi/WithStatement2.txt @@ -0,0 +1,80 @@ +PyFile:WithStatement2.py + PsiComment(Py:END_OF_LINE_COMMENT)('# with = 1 # legal identifier') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# with = 1') + PsiWhiteSpace('\n') + PyAssignmentStatement + PyTargetExpression: with + PsiElement(Py:IDENTIFIER)('with') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# from __future__ import with_statement, nested_scopes') + PsiWhiteSpace('\n') + PyFromImportStatement + PsiElement(Py:FROM_KEYWORD)('from') + PsiWhiteSpace(' ') + PyReferenceExpression: __future__ + PsiElement(Py:IDENTIFIER)('__future__') + PsiWhiteSpace(' ') + PsiElement(Py:IMPORT_KEYWORD)('import') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: with_statement + PsiElement(Py:IDENTIFIER)('with_statement') + PsiElement(Py:COMMA)(',') + PsiWhiteSpace(' ') + PyImportElement + PyReferenceExpression: nested_scopes + PsiElement(Py:IDENTIFIER)('nested_scopes') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# with x.y(z)[t] as y: pass') + PsiWhiteSpace('\n') + PyWithStatement + PsiElement(Py:WITH_KEYWORD)('with') + PsiWhiteSpace(' ') + PySubscriptionExpression + PyCallExpression: y + PyReferenceExpression: y + PyReferenceExpression: x + PsiElement(Py:IDENTIFIER)('x') + PsiElement(Py:DOT)('.') + PsiElement(Py:IDENTIFIER)('y') + PyArgumentList + PsiElement(Py:LPAR)('(') + PyReferenceExpression: z + PsiElement(Py:IDENTIFIER)('z') + PsiElement(Py:RPAR)(')') + PsiElement(Py:LBRACKET)('[') + PyReferenceExpression: t + PsiElement(Py:IDENTIFIER)('t') + PsiElement(Py:RBRACKET)(']') + PsiWhiteSpace(' ') + PsiElement(Py:AS_KEYWORD)('as') + PsiWhiteSpace(' ') + PyReferenceExpression: y + PsiElement(Py:IDENTIFIER)('y') + PsiElement(Py:COLON)(':') + PsiWhiteSpace(' ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n') + PsiComment(Py:END_OF_LINE_COMMENT)('# with = 1 # now illegal') + PsiWhiteSpace('\n') + PyWithStatement + PsiElement(Py:WITH_KEYWORD)('with') + PsiErrorElement:expression expected + + PsiWhiteSpace(' ') + PyStatementList + PsiElement(Py:EQ)('=') + PsiErrorElement:statement expected, found Py:EQ + + PsiWhiteSpace(' ') + PyExpressionStatement + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index b9c1166ede14..c91a934376bc 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -68,4 +68,13 @@ public class PythonParsingTest extends ParsingTestCase { public void testWithStatement() throws Exception { doTest(true); } + + public void testWithStatement2() throws Exception { + doTest(true); + } + + public void testImportStmt() throws Exception { + doTest(true); + } + }