diff --git a/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java index 2c98fc9d6979..be3802e77329 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java +++ b/python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java @@ -8,8 +8,10 @@ import com.jetbrains.python.PyTokenTypes; * @author yole */ public class PythonIndentingLexer extends PythonIndentingProcessor { + private static final TokenSet TOKENS_TO_MERGE = PyTokenTypes.FSTRING_TEXT_TOKENS; + public PythonIndentingLexer() { - super(new _PythonLexer(null), TokenSet.EMPTY); + super(new _PythonLexer(null), TOKENS_TO_MERGE); } boolean addFinalBreak = true; diff --git a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java index 8fde6cf5760a..0770a0cba7aa 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java +++ b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java @@ -6,9 +6,8 @@ import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.PsiElement; -import com.intellij.psi.PsiErrorElement; -import com.intellij.psi.SyntaxTraverser; import com.intellij.psi.tree.IElementType; +import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.psi.PyElementVisitor; @@ -40,13 +39,8 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor @NotNull @Override public List getLiteralPartRanges() { - final TextRange contentRange = getContentRange(); - return SyntaxTraverser.psiApi() - .children(this) - .filter(child -> PyTokenTypes.FSTRING_TEXT_TOKENS.contains(child.getNode().getElementType())) - .map(PsiElement::getTextRangeInParent) - .map(range -> range.intersection(contentRange)) - .toList(); + final List textTokens = findChildrenByType(PyTokenTypes.FSTRING_TEXT_TOKENS); + return ContainerUtil.map(textTokens, PsiElement::getTextRangeInParent); } @NotNull @@ -85,36 +79,16 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor public List> getDecodedFragments() { final ArrayList> result = new ArrayList<>(); final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(this); - int continuousTextStart = -1; for (PsiElement child = getFirstChild(); child != null; child = child.getNextSibling()) { final IElementType childType = child.getNode().getElementType(); - if (childType == PyTokenTypes.FSTRING_START) { - continue; - } final TextRange relChildRange = child.getTextRangeInParent(); - if (childType == PyElementTypes.FSTRING_FRAGMENT || childType == PyTokenTypes.FSTRING_END) { - if (continuousTextStart != -1) { - result.addAll(decoder.decodeRange(TextRange.create(continuousTextStart, relChildRange.getStartOffset()))); - } - continuousTextStart = -1; - - if (childType == PyElementTypes.FSTRING_FRAGMENT) { - // There shouldn't be any escaping inside interpolated parts - result.add(Pair.create(relChildRange, child.getText())); - } + if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) { + result.addAll(decoder.decodeRange(relChildRange)); } - else if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) { - if (continuousTextStart == -1) { - continuousTextStart = relChildRange.getStartOffset(); - } + else if (childType == PyElementTypes.FSTRING_FRAGMENT) { + // There shouldn't be any escaping inside interpolated parts + result.add(Pair.create(relChildRange, child.getText())); } - else if (!(child instanceof PsiErrorElement)) { - throw new AssertionError("Illegal element " + child + " inside f-string"); - } - } - if (continuousTextStart != -1) { - // There are no closing quotes if we got here - result.addAll(decoder.decodeRange(TextRange.create(continuousTextStart, getTextLength()))); } return result; } diff --git a/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt b/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt index 1b9f3bde7dd6..4ac47a489226 100644 --- a/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt +++ b/python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt @@ -8,8 +8,7 @@ PyFile:FStringSingleSlashBeforeLeftBraceInLiteralPart.py PyStringLiteralExpression: foo\{42} PyFormattedStringElement PsiElement(Py:FSTRING_START)('f'') - PsiElement(Py:FSTRING_TEXT)('foo') - PsiElement(Py:FSTRING_TEXT)('\') + PsiElement(Py:FSTRING_TEXT)('foo\') PyFStringFragment PsiElement(Py:FSTRING_FRAGMENT_START)('{') PyNumericLiteralExpression diff --git a/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt b/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt index 551ff80771fa..fa5e05f205b2 100644 --- a/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt +++ b/python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt @@ -14,14 +14,12 @@ PyFile:FStringSingleSlashesBeforeBracesInFormatPart.py PsiElement(Py:INTEGER_LITERAL)('42') PyFStringFragmentFormatPart PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') - PsiElement(Py:FSTRING_TEXT)('foo') - PsiElement(Py:FSTRING_TEXT)('\') + PsiElement(Py:FSTRING_TEXT)('foo\') PyFStringFragment PsiElement(Py:FSTRING_FRAGMENT_START)('{') PyReferenceExpression: bar PsiElement(Py:IDENTIFIER)('bar') PsiElement(Py:FSTRING_FRAGMENT_END)('}') - PsiElement(Py:FSTRING_TEXT)('baz') - PsiElement(Py:FSTRING_TEXT)('\') + PsiElement(Py:FSTRING_TEXT)('baz\') PsiElement(Py:FSTRING_FRAGMENT_END)('}') PsiElement(Py:FSTRING_END)(''') \ No newline at end of file diff --git a/python/testData/psi/FStringTextTokenMerging.py b/python/testData/psi/FStringTextTokenMerging.py new file mode 100644 index 000000000000..8ddc8b2039ca --- /dev/null +++ b/python/testData/psi/FStringTextTokenMerging.py @@ -0,0 +1,4 @@ +s = f"""foo""bar\"\{rf'green"eggs{42:for"mat}'}\ +baz +quux +""" diff --git a/python/testData/psi/FStringTextTokenMerging.txt b/python/testData/psi/FStringTextTokenMerging.txt new file mode 100644 index 000000000000..2ed5af90dbe8 --- /dev/null +++ b/python/testData/psi/FStringTextTokenMerging.txt @@ -0,0 +1,32 @@ +PyFile:FStringTextTokenMerging.py + PyAssignmentStatement + PyTargetExpression: s + PsiElement(Py:IDENTIFIER)('s') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyStringLiteralExpression: foo""bar"\{rf'green"eggs{42:for"mat}'} +baz +quux + + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('f"""') + PsiElement(Py:FSTRING_TEXT)('foo""bar\"\') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyStringLiteralExpression: green"eggs{42:for"mat} + PyFormattedStringElement + PsiElement(Py:FSTRING_START)('rf'') + PsiElement(Py:FSTRING_RAW_TEXT)('green"eggs') + PyFStringFragment + PsiElement(Py:FSTRING_FRAGMENT_START)('{') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('42') + PyFStringFragmentFormatPart + PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':') + PsiElement(Py:FSTRING_RAW_TEXT)('for"mat') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_END)(''') + PsiElement(Py:FSTRING_FRAGMENT_END)('}') + PsiElement(Py:FSTRING_TEXT)('\\nbaz\nquux\n') + PsiElement(Py:FSTRING_END)('"""') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PyEditingTest.java b/python/testSrc/com/jetbrains/python/PyEditingTest.java index 350d7814b50a..926c2617ea35 100644 --- a/python/testSrc/com/jetbrains/python/PyEditingTest.java +++ b/python/testSrc/com/jetbrains/python/PyEditingTest.java @@ -168,6 +168,20 @@ public class PyEditingTest extends PyTestCase { "\"\"\""); } + // PY-32864 + public void testIndentationOfTripleQuotedFStringContent() { + doTestEnter("if True:\n" + + " s = f\"\"\"\n" + + " SELECT\n" + + "\"\"\"", + "if True:\n" + + " s = f\"\"\"\n" + + " SELECT\n" + + " \n" + + "\"\"\"" + ); + } + public void testOvertypeFromInside() { assertEquals("''", doTestTyping("''", 1, '\'')); } diff --git a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java index 2961a24468c6..9e4bed426504 100644 --- a/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java +++ b/python/testSrc/com/jetbrains/python/PyStringLiteralTest.java @@ -24,6 +24,7 @@ import com.intellij.psi.PsiLanguageInjectionHost; import com.intellij.psi.util.PsiTreeUtil; import com.jetbrains.python.fixtures.PyTestCase; import com.jetbrains.python.psi.PyStringLiteralExpression; +import org.jetbrains.annotations.NotNull; import java.util.ArrayList; import java.util.Arrays; @@ -122,7 +123,7 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals(6, escaper.getOffsetInHost(0, range)); assertEquals(7, escaper.getOffsetInHost(1, range)); // Each \\U0001F600 is represented as a surrogate pair, hence 2 characters-wide step in decoded text - assertEquals(17, escaper.getOffsetInHost(3, range)); + assertEquals(17, escaper.getOffsetInHost(3, range)); assertEquals(27, escaper.getOffsetInHost(5, range)); assertEquals(28, escaper.getOffsetInHost(6, range)); assertEquals(-1, escaper.getOffsetInHost(7, range)); @@ -154,20 +155,30 @@ public class PyStringLiteralTest extends PyTestCase { assertEquals("\n{foo}\r\"", createLiteralFromText("f'\\n{foo}\\r\"'").getStringValue()); } - private static String decodeRange(PyStringLiteralExpression expr, TextRange range) { + public void testFStringDecodedRanges() { + assertContainsOrdered(getCharacterRanges("f'foo\"bar'"), "foo\"bar"); + assertContainsOrdered(getCharacterRanges("f'foo\\'bar'"), "foo", "'", "bar"); + assertContainsOrdered(getCharacterRanges("f'foo\\{bar}'"), "foo\\", "{bar}"); + assertContainsOrdered(getCharacterRanges("f'''foo\nbar'''"), "foo\nbar"); + } + + @NotNull + private static String decodeRange(@NotNull PyStringLiteralExpression expr, @NotNull TextRange range) { final StringBuilder builder = new StringBuilder(); expr.createLiteralTextEscaper().decode(range, builder); return builder.toString(); } - private PyStringLiteralExpression createLiteralFromText(final String text) { + @NotNull + private PyStringLiteralExpression createLiteralFromText(@NotNull String text) { final PsiFile file = PsiFileFactory.getInstance(myFixture.getProject()).createFileFromText("test.py", PythonFileType.INSTANCE, "a = (" + text + ")"); final PyStringLiteralExpression expr = PsiTreeUtil.getParentOfType(file.findElementAt(6), PyStringLiteralExpression.class); assert expr != null; return expr; } - private List getCharacterRanges(String text) { + @NotNull + private List getCharacterRanges(@NotNull String text) { final PyStringLiteralExpression expr = createLiteralFromText(text); assertNotNull(expr); final List characters = new ArrayList<>(); diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java index 5965f370195e..38966a00cd35 100644 --- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java @@ -415,14 +415,14 @@ public class PythonLexerTest extends PyLexerTestCase { public void testFStringUnmatchedQuotesAsTextParts() { doTest("s = f'foo\"bar'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", - "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); + "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); } public void testFStringUnmatchedLineBreaksAsTextParts() { doTest("s = f'''foo\n" + "bar'''", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", - "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); + "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); } public void testFStringNamedUnicodeEscapes() { @@ -440,15 +440,15 @@ public class PythonLexerTest extends PyLexerTestCase { public void testFStringBackslashEscapedBraces() { doTest("s = f'foo\\{x}'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", - "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", + "Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); doTest("s = f'{x:foo\\{y}bar\\}'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", - "Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", + "Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END", - "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", + "Py:FSTRING_TEXT", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK"); } diff --git a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java index 86a130cd9c88..d686622cf6cb 100644 --- a/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java @@ -868,6 +868,10 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON36); } + public void testFStringTextTokenMerging() { + doTest(LanguageLevel.PYTHON36); + } + // PY-19036 public void testAwaitInNonAsyncNestedFunction() { doTest(LanguageLevel.PYTHON35);