MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-32864 Automatically merge f-string text tokens

Browse Source 
It not only makes AST of f-string nodes simpler and more obvious to work with,
but, in general, is also better supported by various platform functionality that
assumes that raw text parts of string literals are not broken into multiple
elements.

GitOrigin-RevId: 931d1ea4c09d145e763aed839dcb4acbb3e43ec7
This commit is contained in:
Mikhail Golubev
2020-08-17 11:12:03 +00:00
committed by intellij-monorepo-bot
parent bfaa1fb147
commit ed4b1332dc
10 changed files with 88 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python-psi-impl/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
+3 -1
View File
@@ -8,8 +8,10 @@ import com.jetbrains.python.PyTokenTypes;
* @author yole
*/
public class PythonIndentingLexer extends PythonIndentingProcessor {
private static final TokenSet TOKENS_TO_MERGE = PyTokenTypes.FSTRING_TEXT_TOKENS;
public PythonIndentingLexer() {
super(new _PythonLexer(null), TokenSet.EMPTY);
super(new _PythonLexer(null), TOKENS_TO_MERGE);
}
boolean addFinalBreak = true;
python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFormattedStringElementImpl.java
+8 -34
View File
@@ -6,9 +6,8 @@ import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiErrorElement;
import com.intellij.psi.SyntaxTraverser;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.containers.ContainerUtil;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.psi.PyElementVisitor;
@@ -40,13 +39,8 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor
@NotNull
@Override
public List<TextRange> getLiteralPartRanges() {
final TextRange contentRange = getContentRange();
return SyntaxTraverser.psiApi()
.children(this)
.filter(child -> PyTokenTypes.FSTRING_TEXT_TOKENS.contains(child.getNode().getElementType()))
.map(PsiElement::getTextRangeInParent)
.map(range -> range.intersection(contentRange))
.toList();
final List<PsiElement> textTokens = findChildrenByType(PyTokenTypes.FSTRING_TEXT_TOKENS);
return ContainerUtil.map(textTokens, PsiElement::getTextRangeInParent);
}
@NotNull
@@ -85,36 +79,16 @@ public class PyFormattedStringElementImpl extends PyElementImpl implements PyFor
public List<Pair<TextRange, String>> getDecodedFragments() {
final ArrayList<Pair<TextRange, String>> result = new ArrayList<>();
final PyStringLiteralDecoder decoder = new PyStringLiteralDecoder(this);
int continuousTextStart = -1;
for (PsiElement child = getFirstChild(); child != null; child = child.getNextSibling()) {
final IElementType childType = child.getNode().getElementType();
if (childType == PyTokenTypes.FSTRING_START) {
continue;
}
final TextRange relChildRange = child.getTextRangeInParent();
if (childType == PyElementTypes.FSTRING_FRAGMENT || childType == PyTokenTypes.FSTRING_END) {
if (continuousTextStart != -1) {
result.addAll(decoder.decodeRange(TextRange.create(continuousTextStart, relChildRange.getStartOffset())));
}
continuousTextStart = -1;
if (childType == PyElementTypes.FSTRING_FRAGMENT) {
// There shouldn't be any escaping inside interpolated parts
result.add(Pair.create(relChildRange, child.getText()));
}
if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) {
result.addAll(decoder.decodeRange(relChildRange));
}
else if (PyTokenTypes.FSTRING_TEXT_TOKENS.contains(childType)) {
if (continuousTextStart == -1) {
continuousTextStart = relChildRange.getStartOffset();
}
else if (childType == PyElementTypes.FSTRING_FRAGMENT) {
// There shouldn't be any escaping inside interpolated parts
result.add(Pair.create(relChildRange, child.getText()));
}
else if (!(child instanceof PsiErrorElement)) {
throw new AssertionError("Illegal element " + child + " inside f-string");
}
}
if (continuousTextStart != -1) {
// There are no closing quotes if we got here
result.addAll(decoder.decodeRange(TextRange.create(continuousTextStart, getTextLength())));
}
return result;
}
python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt
+1 -2
View File
@@ -8,8 +8,7 @@ PyFile:FStringSingleSlashBeforeLeftBraceInLiteralPart.py
PyStringLiteralExpression: foo\{42}
PyFormattedStringElement
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PsiElement(Py:FSTRING_TEXT)('\')
PsiElement(Py:FSTRING_TEXT)('foo\')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt
+2 -4
View File
@@ -14,14 +14,12 @@ PyFile:FStringSingleSlashesBeforeBracesInFormatPart.py
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_TEXT)('foo')
PsiElement(Py:FSTRING_TEXT)('\')
PsiElement(Py:FSTRING_TEXT)('foo\')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: bar
PsiElement(Py:IDENTIFIER)('bar')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('baz')
PsiElement(Py:FSTRING_TEXT)('\')
PsiElement(Py:FSTRING_TEXT)('baz\')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringTextTokenMerging.py
+4
View File
@@ -0,0 +1,4 @@
s = f"""foo""bar\"\{rf'green"eggs{42:for"mat}'}\
baz
quux
"""
python/testData/psi/FStringTextTokenMerging.txt
+32
View File
@@ -0,0 +1,32 @@
PyFile:FStringTextTokenMerging.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo""bar"\{rf'green"eggs{42:for"mat}'}
baz
quux
PyFormattedStringElement
PsiElement(Py:FSTRING_START)('f"""')
PsiElement(Py:FSTRING_TEXT)('foo""bar\"\')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: green"eggs{42:for"mat}
PyFormattedStringElement
PsiElement(Py:FSTRING_START)('rf'')
PsiElement(Py:FSTRING_RAW_TEXT)('green"eggs')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_RAW_TEXT)('for"mat')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('\\nbaz\nquux\n')
PsiElement(Py:FSTRING_END)('"""')
python/testSrc/com/jetbrains/python/PyEditingTest.java
+14
View File
@@ -168,6 +168,20 @@ public class PyEditingTest extends PyTestCase {
"<caret>\"\"\"");
}
// PY-32864
public void testIndentationOfTripleQuotedFStringContent() {
doTestEnter("if True:\n" +
" s = f\"\"\"\n" +
" SELECT<caret>\n" +
"\"\"\"",
"if True:\n" +
" s = f\"\"\"\n" +
" SELECT\n" +
" <caret>\n" +
"\"\"\""
);
}
public void testOvertypeFromInside() {
assertEquals("''", doTestTyping("''", 1, '\''));
}
python/testSrc/com/jetbrains/python/PyStringLiteralTest.java
+15 -4
View File
@@ -24,6 +24,7 @@ import com.intellij.psi.PsiLanguageInjectionHost;
import com.intellij.psi.util.PsiTreeUtil;
import com.jetbrains.python.fixtures.PyTestCase;
import com.jetbrains.python.psi.PyStringLiteralExpression;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.Arrays;
@@ -122,7 +123,7 @@ public class PyStringLiteralTest extends PyTestCase {
assertEquals(6, escaper.getOffsetInHost(0, range));
assertEquals(7, escaper.getOffsetInHost(1, range));
// Each \\U0001F600 is represented as a surrogate pair, hence 2 characters-wide step in decoded text
assertEquals(17, escaper.getOffsetInHost(3, range));
assertEquals(17, escaper.getOffsetInHost(3, range));
assertEquals(27, escaper.getOffsetInHost(5, range));
assertEquals(28, escaper.getOffsetInHost(6, range));
assertEquals(-1, escaper.getOffsetInHost(7, range));
@@ -154,20 +155,30 @@ public class PyStringLiteralTest extends PyTestCase {
assertEquals("\n{foo}\r\"", createLiteralFromText("f'\\n{foo}\\r\"'").getStringValue());
}
private static String decodeRange(PyStringLiteralExpression expr, TextRange range) {
public void testFStringDecodedRanges() {
assertContainsOrdered(getCharacterRanges("f'foo\"bar'"), "foo\"bar");
assertContainsOrdered(getCharacterRanges("f'foo\\'bar'"), "foo", "'", "bar");
assertContainsOrdered(getCharacterRanges("f'foo\\{bar}'"), "foo\\", "{bar}");
assertContainsOrdered(getCharacterRanges("f'''foo\nbar'''"), "foo\nbar");
}
@NotNull
private static String decodeRange(@NotNull PyStringLiteralExpression expr, @NotNull TextRange range) {
final StringBuilder builder = new StringBuilder();
expr.createLiteralTextEscaper().decode(range, builder);
return builder.toString();
}
private PyStringLiteralExpression createLiteralFromText(final String text) {
@NotNull
private PyStringLiteralExpression createLiteralFromText(@NotNull String text) {
final PsiFile file = PsiFileFactory.getInstance(myFixture.getProject()).createFileFromText("test.py", PythonFileType.INSTANCE, "a = (" + text + ")");
final PyStringLiteralExpression expr = PsiTreeUtil.getParentOfType(file.findElementAt(6), PyStringLiteralExpression.class);
assert expr != null;
return expr;
}
private List<String> getCharacterRanges(String text) {
@NotNull
private List<String> getCharacterRanges(@NotNull String text) {
final PyStringLiteralExpression expr = createLiteralFromText(text);
assertNotNull(expr);
final List<String> characters = new ArrayList<>();
python/testSrc/com/jetbrains/python/PythonLexerTest.java
+5 -5
View File
@@ -415,14 +415,14 @@ public class PythonLexerTest extends PyLexerTestCase {
public void testFStringUnmatchedQuotesAsTextParts() {
doTest("s = f'foo\"bar'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
}
public void testFStringUnmatchedLineBreaksAsTextParts() {
doTest("s = f'''foo\n" +
"bar'''",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
}
public void testFStringNamedUnicodeEscapes() {
@@ -440,15 +440,15 @@ public class PythonLexerTest extends PyLexerTestCase {
public void testFStringBackslashEscapedBraces() {
doTest("s = f'foo\\{x}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT",
"Py:FSTRING_START", "Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END",
"Py:FSTRING_END", "Py:STATEMENT_BREAK");
doTest("s = f'{x:foo\\{y}bar\\}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER",
"Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END",
"Py:FSTRING_TEXT", "Py:FSTRING_TEXT",
"Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
}
python/testSrc/com/jetbrains/python/parsing/PythonParsingTest.java
+4
View File
@@ -868,6 +868,10 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringTextTokenMerging() {
doTest(LanguageLevel.PYTHON36);
}
// PY-19036
public void testAwaitInNonAsyncNestedFunction() {
doTest(LanguageLevel.PYTHON35);