From 5601b6270c5a82ec1f5473010417457cd306eb70 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Fri, 31 Aug 2018 19:43:34 +0300 Subject: [PATCH] PY-31442 Handle multiline expression fragments inside all-triple-quoted f-strings --- .../python/lexer/PyLexerFStringHelper.kt | 7 +-- .../lexer/PythonIndentingProcessor.java | 33 +++++++++++++- ...ineFStringContainingMultilineExpression.py | 2 + ...neFStringContainingMultilineExpression.txt | 23 ++++++++++ ...ltilineFStringsWithMultilineExpressions.py | 5 +++ ...tilineFStringsWithMultilineExpressions.txt | 34 ++++++++++++++ ...StringTerminatedByLineBreakInExpression.py | 2 + ...tringTerminatedByLineBreakInExpression.txt | 40 +++++++++++++++++ ...tedByLineBreakInExpressionInParentheses.py | 2 + ...edByLineBreakInExpressionInParentheses.txt | 45 +++++++++++++++++++ ...ilineFStringTerminatedByLineBreakInText.py | 2 + ...lineFStringTerminatedByLineBreakInText.txt | 31 +++++++++++++ .../jetbrains/python/PythonParsingTest.java | 20 +++++++++ 13 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 python/testData/psi/MultilineFStringContainingMultilineExpression.py create mode 100644 python/testData/psi/MultilineFStringContainingMultilineExpression.txt create mode 100644 python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.py create mode 100644 python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.txt create mode 100644 python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.py create mode 100644 python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.txt create mode 100644 python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.py create mode 100644 python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.txt create mode 100644 python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.py create mode 100644 python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.txt diff --git a/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt b/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt index ccafbfd1e3e6..402d62ff7588 100644 --- a/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt +++ b/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt @@ -103,10 +103,11 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) { continue } if (c == '\n') { - val firstSingleQuotedIndex = myFStringStates.indexOfFirst { it.openingQuotes.length == 1 } - if (firstSingleQuotedIndex >= 0) { + val insideSingleQuoted = myFStringStates.any { it.openingQuotes.length == 1 } + if (insideSingleQuoted) { if (i == 0) { - dropFStringStateWithAllNested(firstSingleQuotedIndex) + // Terminate all f-strings and insert STATEMENT_BREAK at this point + dropFStringStateWithAllNested(0) } pushBackToOrConsumeMatch(i, 1) return Pair(PyTokenTypes.LINE_BREAK, i) diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index 2389ec9fd4ff..13db518df37a 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -20,8 +20,11 @@ import com.intellij.lexer.FlexLexer; import com.intellij.lexer.MergingLexerAdapter; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.containers.Stack; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.PythonDialectsTokenSetProvider; +import com.jetbrains.python.psi.PyStringLiteralUtil; import gnu.trove.TIntStack; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -35,10 +38,13 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { protected boolean myLineHasSignificantTokens; protected int myLastNewLineIndent = -1; private int myCurrentNewLineIndent = 0; + protected List myTokenQueue = new ArrayList<>(); private int myLineBreakBeforeFirstCommentIndex = -1; protected boolean myProcessSpecialTokensPending = false; + private final Stack myFStringStack = new Stack<>(); + private static final boolean DUMP_TOKENS = false; private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens(); @@ -179,6 +185,23 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { protected void advanceBase() { super.advance(); checkSignificantTokens(); + checkFString(); + } + + private void checkFString() { + final String tokenText = getTokenText(); + if (isBaseAt(PyTokenTypes.FSTRING_START)) { + final int prefixLength = PyStringLiteralUtil.getPrefixLength(tokenText); + myFStringStack.push(tokenText.substring(prefixLength)); + } + else if (isBaseAt(PyTokenTypes.FSTRING_END)) { + while (!myFStringStack.isEmpty()) { + final String lastOpeningQuotes = myFStringStack.pop(); + if (lastOpeningQuotes.equals(tokenText)) { + break; + } + } + } } protected void pushToken(IElementType type, int start, int end) { @@ -205,6 +228,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { adjustBraceLevel(); myLineHasSignificantTokens = false; checkSignificantTokens(); + checkFString(); if (isBaseAt(PyTokenTypes.SPACE)) { processIndent(0, PyTokenTypes.SPACE); } @@ -311,9 +335,14 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { } protected void processLineBreak(int startPos) { - if (myBraceLevel == 0) { - if (myLineHasSignificantTokens) { + // See https://www.python.org/dev/peps/pep-0498/#expression-evaluation + final boolean allFStringsAreTripleQuoted = ContainerUtil.and(myFStringStack, quotes -> quotes.length() == 3); + final boolean insideImplicitFragmentParentheses = !myFStringStack.isEmpty() && allFStringsAreTripleQuoted; + final boolean shouldTerminateFStrings = !myFStringStack.isEmpty() && !allFStringsAreTripleQuoted; + if ((myBraceLevel == 0 && !insideImplicitFragmentParentheses) || shouldTerminateFStrings) { + if (myLineHasSignificantTokens || shouldTerminateFStrings) { pushToken(PyTokenTypes.STATEMENT_BREAK, startPos, startPos); + myFStringStack.clear(); } myLineHasSignificantTokens = false; advanceBase(); diff --git a/python/testData/psi/MultilineFStringContainingMultilineExpression.py b/python/testData/psi/MultilineFStringContainingMultilineExpression.py new file mode 100644 index 000000000000..920f3f79d180 --- /dev/null +++ b/python/testData/psi/MultilineFStringContainingMultilineExpression.py @@ -0,0 +1,2 @@ +s = f'''{1 + +2}''' \ No newline at end of file diff --git a/python/testData/psi/MultilineFStringContainingMultilineExpression.txt b/python/testData/psi/MultilineFStringContainingMultilineExpression.txt new file mode 100644 index 000000000000..2f682d935d8b --- /dev/null +++ b/python/testData/psi/MultilineFStringContainingMultilineExpression.txt @@ -0,0 +1,23 @@ +PyFile:MultilineFStringContainingMultilineExpression.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {1 + +2} + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f'''') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyBinaryExpression + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') + PsiWhiteSpace(' ') + PsiElement(Py:PLUS)('+') + PsiWhiteSpace('\n') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('2') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_END)(''''') \ No newline at end of file diff --git a/python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.py b/python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.py new file mode 100644 index 000000000000..dabb5d897ee0 --- /dev/null +++ b/python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.py @@ -0,0 +1,5 @@ +s = f"""{f''' +{"bar" +} +''' +}""" \ No newline at end of file diff --git a/python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.txt b/python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.txt new file mode 100644 index 000000000000..6067cd514dab --- /dev/null +++ b/python/testData/psi/NestedMultilineFStringsWithMultilineExpressions.txt @@ -0,0 +1,34 @@ +PyFile:NestedMultilineFStringsWithMultilineExpressions.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {f''' +{"bar" +} +''' +} + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f"""') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: +{"bar" +} + + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f'''') + PsiElement(Py:FSTRING_TEXT)('\n') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: bar + PsiElement(Py:SINGLE_QUOTED_STRING)('"bar"') + PsiWhiteSpace('\n') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_TEXT)('\n') + PsiElement(Py:FSTRING_END)(''''') + PsiWhiteSpace('\n') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_END)('"""') \ No newline at end of file diff --git a/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.py b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.py new file mode 100644 index 000000000000..ebefd5909f5f --- /dev/null +++ b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.py @@ -0,0 +1,2 @@ +s = f"""{f'{1 + +2}'}""" \ No newline at end of file diff --git a/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.txt b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.txt new file mode 100644 index 000000000000..033d829d322e --- /dev/null +++ b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.txt @@ -0,0 +1,40 @@ +PyFile:SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {f'{1 + + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f"""') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: {1 + + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyBinaryExpression + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') + PsiWhiteSpace(' ') + PsiElement(Py:PLUS)('+') + PsiErrorElement:expression expected + + PsiErrorElement:Unexpected f-string token + + PsiErrorElement:Unexpected f-string token + + PsiWhiteSpace(' \n') + PyExpressionStatement + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('2') + PsiErrorElement:End of statement expected + + PsiElement(Py:RBRACE)('}') + PsiErrorElement:Statement expected, found Py:RBRACE + + PyExpressionStatement + PyStringLiteralExpression: }""" + PsiElement(Py:SINGLE_QUOTED_STRING)(''}"""') \ No newline at end of file diff --git a/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.py b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.py new file mode 100644 index 000000000000..d7ff1613c1f5 --- /dev/null +++ b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.py @@ -0,0 +1,2 @@ +s = f"""{f'{(1 + +2)}'}""" \ No newline at end of file diff --git a/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.txt b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.txt new file mode 100644 index 000000000000..0069dd973f61 --- /dev/null +++ b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.txt @@ -0,0 +1,45 @@ +PyFile:SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {f'{(1 + + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f"""') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: {(1 + + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f'') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyParenthesizedExpression + PsiElement(Py:LPAR)('(') + PyBinaryExpression + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') + PsiWhiteSpace(' ') + PsiElement(Py:PLUS)('+') + PsiErrorElement:expression expected + + PsiErrorElement:Unexpected f-string token + + PsiErrorElement:Unexpected f-string token + + PsiWhiteSpace(' \n') + PyExpressionStatement + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('2') + PsiErrorElement:End of statement expected + + PsiElement(Py:RPAR)(')') + PsiErrorElement:Statement expected, found Py:RPAR + + PsiElement(Py:RBRACE)('}') + PsiErrorElement:Statement expected, found Py:RBRACE + + PyExpressionStatement + PyStringLiteralExpression: }""" + PsiElement(Py:SINGLE_QUOTED_STRING)(''}"""') \ No newline at end of file diff --git a/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.py b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.py new file mode 100644 index 000000000000..ceaafa9245f7 --- /dev/null +++ b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.py @@ -0,0 +1,2 @@ +s = f"""{f'foo +bar'}""" \ No newline at end of file diff --git a/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.txt b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.txt new file mode 100644 index 000000000000..0309cb2cc36d --- /dev/null +++ b/python/testData/psi/SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.txt @@ -0,0 +1,31 @@ +PyFile:SingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: {f'foo + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f"""') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: foo + PyFormattedStringNode + PsiElement(Py:FSTRING_START)('f'') + PsiElement(Py:FSTRING_TEXT)('foo') + PsiErrorElement:Unexpected f-string token + + PsiErrorElement:type conversion, : or } expected + + PsiErrorElement:Unexpected f-string token + + PsiWhiteSpace('\n') + PyExpressionStatement + PyReferenceExpression: bar + PsiElement(Py:IDENTIFIER)('bar') + PsiErrorElement:End of statement expected + + PyExpressionStatement + PyStringLiteralExpression: }""" + PsiElement(Py:SINGLE_QUOTED_STRING)(''}"""') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 7ca4915f82c6..89baf7df4354 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -727,6 +727,26 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON36); } + public void testMultilineFStringContainingMultilineExpression() { + doTest(LanguageLevel.PYTHON36); + } + + public void testNestedMultilineFStringsWithMultilineExpressions() { + doTest(LanguageLevel.PYTHON36); + } + + public void testSingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInText() { + doTest(LanguageLevel.PYTHON36); + } + + public void testSingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpression() { + doTest(LanguageLevel.PYTHON36); + } + + public void testSingleQuotedFStringInsideMultilineFStringTerminatedByLineBreakInExpressionInParentheses() { + doTest(LanguageLevel.PYTHON36); + } + // PY-19036 public void testAwaitInNonAsyncNestedFunction() { doTest(LanguageLevel.PYTHON35);