From ed4b1332dc86f76bbc1f5bd689c7ad59a2b9e8a5 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Sat, 15 Aug 2020 18:21:49 +0300 Subject: [PATCH] PY-32864 Automatically merge f-string text tokens It not only makes AST of f-string nodes simpler and more obvious to work with, but, in general, is also better supported by various platform functionality that assumes that raw text parts of string literals are not broken into multiple elements. GitOrigin-RevId: 931d1ea4c09d145e763aed839dcb4acbb3e43ec7 --- .../python/lexer/PythonIndentingLexer.java | 4 +- .../impl/PyFormattedStringElementImpl.java | 42 ++++--------------- ...ingleSlashBeforeLeftBraceInLiteralPart.txt | 3 +- ...gSingleSlashesBeforeBracesInFormatPart.txt | 6 +-- .../testData/psi/FStringTextTokenMerging.py | 4 ++ .../testData/psi/FStringTextTokenMerging.txt | 32 ++++++++++++++ .../com/jetbrains/python/PyEditingTest.java | 14 +++++++ .../jetbrains/python/PyStringLiteralTest.java | 19 +++++++-- .../com/jetbrains/python/PythonLexerTest.java | 10 ++--- .../python/parsing/PythonParsingTest.java | 4 ++ 10 files changed, 88 insertions(+), 50 deletions(-) create mode 100644 python/testData/psi/FStringTextTokenMerging.py create mode 100644 python/testData/psi/FStringTextTokenMerging.txt diff --git a/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java index 2c98fc9d6979..be3802e77329 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java +++ b/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java @@ -8,8 +8,10 @@ import com.jetbrains.python.PyTokenTypes; * @author yole */ public class PythonIndentingLexer extends PythonIndentingProcessor { + private static final TokenSet TOKENS_TO_MERGE = PyTokenTypes.FSTRING_TEXT_TOKENS; + public PythonIndentingLexer() { - super(new _PythonLexer(null), TokenSet.EMPTY); + super(new _PythonLexer(null), TOKENS_TO_MERGE); } boolean addFinalBreak = true; diff --git a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java index 8fde6cf5760a..0770a0cba7aa 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java +++ b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java @@ -6,9 +6,8 @@ import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.PsiElement; -import com.intellij.psi.PsiErrorElement; -import com.intellij.psi.SyntaxTraverser; import com.intellij.psi.tree.IElementType; +import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.psi.PyElementVisitor; @@ -40,13 +39,8 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor @NotNull @Override public List getLiteralPartRanges() { - final TextRange contentRange = getContentRange(); - return SyntaxTraverser.psiApi() - .children(this) - .filter(child -> PyTokenTypes.FSTRING_TEXT_TOKENS.contains(child.getNode().getElementType())) - .map(PsiElement::getTextRangeInParent) - .map(range -> range.intersection(contentRange)) - .toList(); + final List textTokens = findChildrenByType(PyTokenTypes.FSTRING_TEXT_TOKENS); + return ContainerUtil.map(textTokens, PsiElement::getTextRangeInParent); } @NotNull @@ -85,36 +79,16 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor public List> getDecodedFragments() { final ArrayList> result = new ArrayList<>(); final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(this); - int continuousTextStart = -1; for (PsiElement child = getFirstChild(); child != null; child = child.getNextSibling()) { final IElementType childType = child.getNode().getElementType(); - if (childType == PyTokenTypes.FSTRING_START) { - continue; - } final TextRange relChildRange = child.getTextRangeInParent(); - if (childType == PyElementTypes.FSTRING_FRAGMENT || childType == PyTokenTypes.FSTRING_END) { - if (continuousTextStart != -1) { - result.addAll(decoder.decodeRange(TextRange.create(continuousTextStart, relChildRange.getStartOffset()))); - } - continuousTextStart = -1; - - if (childType == PyElementTypes.FSTRING_FRAGMENT) { - // There shouldn't be any escaping inside interpolated parts - result.add(Pair.create(relChildRange, child.getText())); - } + if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) { + result.addAll(decoder.decodeRange(relChildRange)); } - else if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) { - if (continuousTextStart == -1) { - continuousTextStart = relChildRange.getStartOffset(); - } + else if (childType == PyElementTypes.FSTRING_FRAGMENT) { + // There shouldn't be any escaping inside interpolated parts + result.add(Pair.create(relChildRange, child.getText())); } - else if (!(child instanceof PsiErrorElement)) { - throw new AssertionError("Illegal element " + child + " inside f-string"); - } - } - if (continuousTextStart != -1) { - // There are no closing quotes if we got here - result.addAll(decoder.decodeRange(TextRange.create(continuousTextStart, getTextLength()))); } return result; } diff --git a/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt b/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt index 1b9f3bde7dd6..4ac47a489226 100644 --- a/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt +++ b/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt @@ -8,8 +8,7 @@ PyFile:FStringSingleSlashBeforeLeftBraceInLiteralPart.py PyStringLiteralExpression: foo\{42} PyFormattedStringElement PsiElement(Py:FSTRING_START)('f'') - PsiElement(Py:FSTRING_TEXT)('foo') - PsiElement(Py:FSTRING_TEXT)('\') + PsiElement(Py:FSTRING_TEXT)('foo\') PyFStringFragment PsiElement(Py:FSTRING_FRAGMENT_START)('{') PyNumericLiteralExpression diff --git a/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt b/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt index 551ff80771fa..fa5e05f205b2 100644 --- a/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt +++ b/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt @@ -14,14 +14,12 @@ PyFile:FStringSingleSlashesBeforeBracesInFormatPart.py PsiElement(Py:INTEGER_LITERAL)('42') PyFStringFragmentFormatPart PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') - PsiElement(Py:FSTRING_TEXT)('foo') - PsiElement(Py:FSTRING_TEXT)('\') + PsiElement(Py:FSTRING_TEXT)('foo\') PyFStringFragment PsiElement(Py:FSTRING_FRAGMENT_START)('{') PyReferenceExpression: bar PsiElement(Py:IDENTIFIER)('bar') PsiElement(Py:FSTRING_FRAGMENT_END)('}') - PsiElement(Py:FSTRING_TEXT)('baz') - PsiElement(Py:FSTRING_TEXT)('\') + PsiElement(Py:FSTRING_TEXT)('baz\') PsiElement(Py:FSTRING_FRAGMENT_END)('}') PsiElement(Py:FSTRING_END)(''') \ No newline at end of file diff --git a/python/testData/psi/FStringTextTokenMerging.py b/python/testData/psi/FStringTextTokenMerging.py new file mode 100644 index 000000000000..8ddc8b2039ca --- /dev/null +++ b/python/testData/psi/FStringTextTokenMerging.py @@ -0,0 +1,4 @@ +s = f"""foo""bar\"\{rf'green"eggs{42:for"mat}'}\ +baz +quux +""" diff --git a/python/testData/psi/FStringTextTokenMerging.txt b/python/testData/psi/FStringTextTokenMerging.txt new file mode 100644 index 000000000000..2ed5af90dbe8 --- /dev/null +++ b/python/testData/psi/FStringTextTokenMerging.txt @@ -0,0 +1,32 @@ +PyFile:FStringTextTokenMerging.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: foo""bar"\{rf'green"eggs{42:for"mat}'} +baz +quux + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f"""') + PsiElement(Py:FSTRING_TEXT)('foo""bar\"\') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: green"eggs{42:for"mat} + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('rf'') + PsiElement(Py:FSTRING_RAW_TEXT)('green"eggs') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('42') + PyFStringFragmentFormatPart + PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') + PsiElement(Py:FSTRING_RAW_TEXT)('for"mat') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_END)(''') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_TEXT)('\\nbaz\nquux\n') + PsiElement(Py:FSTRING_END)('"""') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PyEditingTest.java b/python/testSrc/com/jetbrains/python/PyEditingTest.java index 350d7814b50a..926c2617ea35 100644 --- a/python/testSrc/com/jetbrains/python/PyEditingTest.java +++ b/python/testSrc/com/jetbrains/python/PyEditingTest.java @@ -168,6 +168,20 @@ public class PyEditingTest extends PyTestCase { "\"\"\""); } + // PY-32864 + public void testIndentationOfTripleQuotedFStringContent() { + doTestEnter("if True:\n" + + " s = f\"\"\"\n" + + " SELECT\n" + + "\"\"\"", + "if True:\n" + + " s = f\"\"\"\n" + + " SELECT\n" + + " \n" + + "\"\"\"" + ); + } + public void testOvertypeFromInside() { assertEquals("''", doTestTyping("''", 1, '\'')); } diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java index 2961a24468c6..9e4bed426504 100644 --- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java @@ -24,6 +24,7 @@ import com.intellij.psi.PsiLanguageInjectionHost; import com.intellij.psi.util.PsiTreeUtil; import com.jetbrains.python.fixtures.PyTestCase; import com.jetbrains.python.psi.PyStringLiteralExpression; +import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Arrays; @@ -122,7 +123,7 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals(6, escaper.getOffsetInHost(0, range)); assertEquals(7, escaper.getOffsetInHost(1, range)); // Each \\U0001F600 is represented as a surrogate pair, hence 2 characters-wide step in decoded text - assertEquals(17, escaper.getOffsetInHost(3, range)); + assertEquals(17, escaper.getOffsetInHost(3, range)); assertEquals(27, escaper.getOffsetInHost(5, range)); assertEquals(28, escaper.getOffsetInHost(6, range)); assertEquals(-1, escaper.getOffsetInHost(7, range)); @@ -154,20 +155,30 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals("\n{foo}\r\"", createLiteralFromText("f'\\n{foo}\\r\"'").getStringValue()); } - private static String decodeRange(PyStringLiteralExpression expr, TextRange range) { + public void testFStringDecodedRanges() { + assertContainsOrdered(getCharacterRanges("f'foo\"bar'"), "foo\"bar"); + assertContainsOrdered(getCharacterRanges("f'foo\\'bar'"), "foo", "'", "bar"); + assertContainsOrdered(getCharacterRanges("f'foo\\{bar}'"), "foo\\", "{bar}"); + assertContainsOrdered(getCharacterRanges("f'''foo\nbar'''"), "foo\nbar"); + } + + @NotNull + private static String decodeRange(@NotNull PyStringLiteralExpression expr, @NotNull TextRange range) { final StringBuilder builder = new StringBuilder(); expr.createLiteralTextEscaper().decode(range, builder); return builder.toString(); } - private PyStringLiteralExpression createLiteralFromText(final String text) { + @NotNull + private PyStringLiteralExpression createLiteralFromText(@NotNull String text) { final PsiFile file = PsiFileFactory.getInstance(myFixture.getProject()).createFileFromText("test.py", PythonFileType.INSTANCE, "a = (" + text + ")"); final PyStringLiteralExpression expr = PsiTreeUtil.getParentOfType(file.findElementAt(6), PyStringLiteralExpression.class); assert expr != null; return expr; } - private List getCharacterRanges(String text) { + @NotNull + private List getCharacterRanges(@NotNull String text) { final PyStringLiteralExpression expr = createLiteralFromText(text); assertNotNull(expr); final List characters = new ArrayList<>(); diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java index 5965f370195e..38966a00cd35 100644 --- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java @@ -415,14 +415,14 @@ public class PythonLexerTest extends PyLexerTestCase { public void testFStringUnmatchedQuotesAsTextParts() { doTest("s = f'foo\"bar'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", - "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); + "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); } public void testFStringUnmatchedLineBreaksAsTextParts() { doTest("s = f'''foo\n" + "bar'''", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", - "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); + "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); } public void testFStringNamedUnicodeEscapes() { @@ -440,15 +440,15 @@ public class PythonLexerTest extends PyLexerTestCase { public void testFStringBackslashEscapedBraces() { doTest("s = f'foo\\{x}'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", - "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); doTest("s = f'{x:foo\\{y}bar\\}'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", - "Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", + "Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END", - "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", + "Py:FSTRING_TEXT", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); } diff --git a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java index 86a130cd9c88..d686622cf6cb 100644 --- a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java @@ -868,6 +868,10 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON36); } + public void testFStringTextTokenMerging() { + doTest(LanguageLevel.PYTHON36); + } + // PY-19036 public void testAwaitInNonAsyncNestedFunction() { doTest(LanguageLevel.PYTHON35);