From db5297d5e13c38ef75a8416689b62197cb945d45 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Wed, 23 Mar 2011 15:12:14 +0100 Subject: [PATCH] change parsing of generator in argument list so that parentheses are part of argument list, not generator (PY-3172) --- .../python/parsing/ExpressionParsing.java | 26 ++++++++++++------- .../testData/formatter/noWrapBeforeParen.py | 6 +++++ .../formatter/noWrapBeforeParen_after.py | 7 +++++ .../testData/psi/GeneratorInArgumentList.py | 1 + .../testData/psi/GeneratorInArgumentList.txt | 24 +++++++++++++++++ .../com/jetbrains/python/PyFormatterTest.java | 4 +++ .../jetbrains/python/PythonParsingTest.java | 4 +++ 7 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 python/testData/formatter/noWrapBeforeParen.py create mode 100644 python/testData/formatter/noWrapBeforeParen_after.py create mode 100644 python/testData/psi/GeneratorInArgumentList.py create mode 100644 python/testData/psi/GeneratorInArgumentList.txt diff --git a/python/src/com/jetbrains/python/parsing/ExpressionParsing.java b/python/src/com/jetbrains/python/parsing/ExpressionParsing.java index 9a7e343307cd..3f5f17e3e3b3 100644 --- a/python/src/com/jetbrains/python/parsing/ExpressionParsing.java +++ b/python/src/com/jetbrains/python/parsing/ExpressionParsing.java @@ -99,7 +99,7 @@ public class ExpressionParsing extends Parsing { builder.error(message("PARSE.expected.expression")); } if (builder.getTokenType() == PyTokenTypes.FOR_KEYWORD) { - parseComprehension(expr, PyTokenTypes.RBRACKET, PyElementTypes.LIST_COMP_EXPRESSION); + parseComprehension(expr, PyTokenTypes.RBRACKET, PyElementTypes.LIST_COMP_EXPRESSION, false); } else { while (builder.getTokenType() != PyTokenTypes.RBRACKET) { @@ -120,8 +120,9 @@ public class ExpressionParsing extends Parsing { } private void parseComprehension(PsiBuilder.Marker expr, - final IElementType endToken, - final IElementType exprType) { + final IElementType endToken, + final IElementType exprType, + final boolean leaveEndTokenOutside) { assertCurrentToken(PyTokenTypes.FOR_KEYWORD); while (true) { myBuilder.advanceLexer(); @@ -134,8 +135,13 @@ public class ExpressionParsing extends Parsing { myBuilder.advanceLexer(); parseOldExpression(); } - if (myBuilder.getTokenType() == endToken) { - myBuilder.advanceLexer(); + if (atToken(endToken)) { + if (leaveEndTokenOutside) { + expr.done(exprType); + nextToken(); + return; + } + nextToken(); break; } if (myBuilder.getTokenType() == PyTokenTypes.FOR_KEYWORD) { @@ -176,7 +182,7 @@ public class ExpressionParsing extends Parsing { } else if (atToken(PyTokenTypes.FOR_KEYWORD)) { firstExprMarker.drop(); - parseComprehension(expr, PyTokenTypes.RBRACE, PyElementTypes.SET_COMP_EXPRESSION); + parseComprehension(expr, PyTokenTypes.RBRACE, PyElementTypes.SET_COMP_EXPRESSION, false); } else { myBuilder.error("expression expected"); @@ -197,7 +203,7 @@ public class ExpressionParsing extends Parsing { } firstKeyValueMarker.done(PyElementTypes.KEY_VALUE_EXPRESSION); if (myBuilder.getTokenType() == PyTokenTypes.FOR_KEYWORD) { - parseComprehension(startMarker, PyTokenTypes.RBRACE, PyElementTypes.DICT_COMP_EXPRESSION); + parseComprehension(startMarker, PyTokenTypes.RBRACE, PyElementTypes.DICT_COMP_EXPRESSION, false); } else { while (myBuilder.getTokenType() != PyTokenTypes.RBRACE) { @@ -249,7 +255,7 @@ public class ExpressionParsing extends Parsing { else { parseYieldOrTupleExpression(isTargetExpression); if (myBuilder.getTokenType() == PyTokenTypes.FOR_KEYWORD) { - parseComprehension(expr, PyTokenTypes.RPAR, PyElementTypes.GENERATOR_EXPRESSION); + parseComprehension(expr, PyTokenTypes.RPAR, PyElementTypes.GENERATOR_EXPRESSION, false); } else { checkMatches(PyTokenTypes.RPAR, message("PARSE.expected.rpar")); @@ -409,15 +415,15 @@ public class ExpressionParsing extends Parsing { public void parseArgumentList() { LOG.assertTrue(myBuilder.getTokenType() == PyTokenTypes.LPAR); final PsiBuilder.Marker arglist = myBuilder.mark(); - final PsiBuilder.Marker genexpr = myBuilder.mark(); myBuilder.advanceLexer(); + final PsiBuilder.Marker genexpr = myBuilder.mark(); int argNumber = 0; boolean needBracket = true; while (myBuilder.getTokenType() != PyTokenTypes.RPAR) { argNumber++; if (argNumber > 1) { if (argNumber == 2 && myBuilder.getTokenType() == PyTokenTypes.FOR_KEYWORD && genexpr != null) { - parseComprehension(genexpr, PyTokenTypes.RPAR, PyElementTypes.GENERATOR_EXPRESSION); + parseComprehension(genexpr, PyTokenTypes.RPAR, PyElementTypes.GENERATOR_EXPRESSION, true); needBracket = false; break; } diff --git a/python/testData/formatter/noWrapBeforeParen.py b/python/testData/formatter/noWrapBeforeParen.py new file mode 100644 index 000000000000..9f7839338079 --- /dev/null +++ b/python/testData/formatter/noWrapBeforeParen.py @@ -0,0 +1,6 @@ +def foo(): + def bar(): + def baz(): + if comments: + for comment in comments: + record += ' \n' \ No newline at end of file diff --git a/python/testData/formatter/noWrapBeforeParen_after.py b/python/testData/formatter/noWrapBeforeParen_after.py new file mode 100644 index 000000000000..61f5d746a00e --- /dev/null +++ b/python/testData/formatter/noWrapBeforeParen_after.py @@ -0,0 +1,7 @@ +def foo(): + def bar(): + def baz(): + if comments: + for comment in comments: + record += ' \n' \ No newline at end of file diff --git a/python/testData/psi/GeneratorInArgumentList.py b/python/testData/psi/GeneratorInArgumentList.py new file mode 100644 index 000000000000..c0e4b32d9ebe --- /dev/null +++ b/python/testData/psi/GeneratorInArgumentList.py @@ -0,0 +1 @@ +"".join(' ' for ca in comment) \ No newline at end of file diff --git a/python/testData/psi/GeneratorInArgumentList.txt b/python/testData/psi/GeneratorInArgumentList.txt new file mode 100644 index 000000000000..0a15e9a61b27 --- /dev/null +++ b/python/testData/psi/GeneratorInArgumentList.txt @@ -0,0 +1,24 @@ +PyFile:GeneratorInArgumentList.py + PyExpressionStatement + PyCallExpression: "".join + PyReferenceExpression: join + PyStringLiteralExpression: + PsiElement(Py:STRING_LITERAL)('""') + PsiElement(Py:DOT)('.') + PsiElement(Py:IDENTIFIER)('join') + PyArgumentList + PsiElement(Py:LPAR)('(') + PyGeneratorExpression + PyStringLiteralExpression: + PsiElement(Py:STRING_LITERAL)('' '') + PsiWhiteSpace(' ') + PsiElement(Py:FOR_KEYWORD)('for') + PsiWhiteSpace(' ') + PyTargetExpression: ca + PsiElement(Py:IDENTIFIER)('ca') + PsiWhiteSpace(' ') + PsiElement(Py:IN_KEYWORD)('in') + PsiWhiteSpace(' ') + PyReferenceExpression: comment + PsiElement(Py:IDENTIFIER)('comment') + PsiElement(Py:RPAR)(')') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PyFormatterTest.java b/python/testSrc/com/jetbrains/python/PyFormatterTest.java index ecb58ad554ae..c11e9395ba30 100644 --- a/python/testSrc/com/jetbrains/python/PyFormatterTest.java +++ b/python/testSrc/com/jetbrains/python/PyFormatterTest.java @@ -104,6 +104,10 @@ public class PyFormatterTest extends PyLightFixtureTestCase { doTest(); } + public void testNoWrapBeforeParen() { // PY-3172 + doTest(); + } + private void doTest() { myFixture.configureByFile("formatter/" + getTestName(true) + ".py"); ApplicationManager.getApplication().runWriteAction(new Runnable() { diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 9656fcd6290c..bfcd853a8add 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -255,6 +255,10 @@ public class PythonParsingTest extends ParsingTestCase { doTest(); } + public void testGeneratorInArgumentList() { // PY-3172 + doTest(); + } + public void doTest() { doTest(LanguageLevel.PYTHON25); }