MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2025-12-14 09:12:22 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-31442 Better recovery inside expression fragments

Browse Source 
Including cases when there is a dangling backslash inside an expression.
This commit is contained in:
Mikhail Golubev
2018-09-03 15:55:55 +03:00
parent 36fb61ffbb
commit 77e47676b4
17 changed files with 215 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
python/src/com/jetbrains/python/parsing/ExpressionParsing.java
View File

@@ -151,11 +151,31 @@ public class ExpressionParsing extends Parsing {
}
private void parseFStringFragment() {
PsiBuilder builder = myContext.getBuilder();
final PsiBuilder builder = myContext.getBuilder();
if (atToken(PyTokenTypes.FSTRING_FRAGMENT_START)) {
final PsiBuilder.Marker marker = builder.mark();
nextToken();
myContext.getExpressionParser().parseExpression();
PsiBuilder.Marker recoveryMarker = builder.mark();
final boolean parsedExpression = myContext.getExpressionParser().parseExpressionOptional();
if (parsedExpression) {
recoveryMarker.drop();
recoveryMarker = builder.mark();
}
boolean recovery = !parsedExpression;
while (!builder.eof() && !atAnyOfTokens(PyTokenTypes.FSTRING_FRAGMENT_TYPE_CONVERSION,
PyTokenTypes.FSTRING_FRAGMENT_FORMAT_START,
PyTokenTypes.FSTRING_FRAGMENT_END,
PyTokenTypes.FSTRING_END,
PyTokenTypes.STATEMENT_BREAK)) {
nextToken();
recovery = true;
}
if (recovery) {
recoveryMarker.error(parsedExpression ? "unexpected expression part" : "expression expected");
}
else {
recoveryMarker.drop();
}
final boolean hasTypeConversion = matchToken(PyTokenTypes.FSTRING_FRAGMENT_TYPE_CONVERSION);
final boolean hasFormatPart = atToken(PyTokenTypes.FSTRING_FRAGMENT_FORMAT_START);
if (hasFormatPart) {

1
python/testData/psi/FStringBackslashAfterExpression.py Normal file
View File

@@ -0,0 +1 @@
s = f'foo{42 \ }bar'

21
python/testData/psi/FStringBackslashAfterExpression.txt Normal file
View File

@@ -0,0 +1,21 @@
PyFile:FStringBackslashAfterExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo{42 \ }bar
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiErrorElement:unexpected expression part
PsiElement(Py:BACKSLASH)(' \')
PsiWhiteSpace(' ')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('bar')
PsiElement(Py:FSTRING_END)(''')

1
python/testData/psi/FStringBackslashBeforeExpression.py Normal file
View File

@@ -0,0 +1 @@
s = f'foo{\ 42}bar'

20
python/testData/psi/FStringBackslashBeforeExpression.txt Normal file
View File

@@ -0,0 +1,20 @@
PyFile:FStringBackslashBeforeExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo{\ 42}bar
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
PsiElement(Py:BACKSLASH)('\')
PsiWhiteSpace(' ')
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('bar')
PsiElement(Py:FSTRING_END)(''')

2
python/testData/psi/FStringBackslashInsideMultilineExpression.py Normal file
View File

@@ -0,0 +1,2 @@
s = f"""{ 1 + \
2}"""

25
python/testData/psi/FStringBackslashInsideMultilineExpression.txt Normal file
View File

@@ -0,0 +1,25 @@
PyFile:FStringBackslashInsideMultilineExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: { 1 + \
2}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"""')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiWhiteSpace(' ')
PyBinaryExpression
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(Py:PLUS)('+')
PsiWhiteSpace(' \')
PsiWhiteSpace('\n')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)('"""')

1
python/testData/psi/FStringBackslashInsteadOfExpression.py Normal file
View File

@@ -0,0 +1 @@
s = f'foo{\}bar'

18
python/testData/psi/FStringBackslashInsteadOfExpression.txt Normal file
View File

@@ -0,0 +1,18 @@
PyFile:FStringBackslashInsteadOfExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo{\}bar
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
PsiElement(Py:BACKSLASH)('\')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('bar')
PsiElement(Py:FSTRING_END)(''')

4
python/testData/psi/FStringStatementInsideExpression.py Normal file
View File

@@ -0,0 +1,4 @@
s = f"""{
def func():
pass
}"""

41
python/testData/psi/FStringStatementInsideExpression.txt Normal file
View File

@@ -0,0 +1,41 @@
PyFile:FStringStatementInsideExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {
def func():
pass
}"""
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"""')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiWhiteSpace('\n')
PsiErrorElement:Unexpected expression token
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiErrorElement:Unexpected expression token
PsiElement(Py:IDENTIFIER)('func')
PsiErrorElement:Unexpected expression token
PsiElement(Py:LPAR)('(')
PsiErrorElement:Unexpected expression token
PsiElement(Py:RPAR)(')')
PsiErrorElement:Unexpected expression token
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PsiErrorElement:Unexpected expression token
PsiElement(Py:PASS_KEYWORD)('pass')
PsiWhiteSpace('\n')
PsiErrorElement:Unexpected expression token
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Unexpected expression token
PsiElement(Py:TRIPLE_QUOTED_STRING)('"""')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>

2
python/testData/psi/FStringTerminatedByQuoteOfNestedStringLiteral.txt
View File

@@ -17,6 +17,8 @@ PyFile:FStringTerminatedByQuoteOfNestedStringLiteral.py
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:Expected "
<empty list>
PsiElement(Py:FSTRING_END)(''')

2
python/testData/psi/FStringTerminatedByQuoteOfNestedStringLiteralInFormatPart.txt
View File

@@ -23,6 +23,8 @@ PyFile:FStringTerminatedByQuoteOfNestedStringLiteralInFormatPart.py
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:Expected "
<empty list>
PsiElement(Py:FSTRING_END)(''')

2
python/testData/psi/FStringTerminatedByQuoteOfStringLiteral.txt
View File

@@ -13,6 +13,8 @@ PyFile:FStringTerminatedByQuoteOfStringLiteral.py
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
PsiErrorElement:End of statement expected
<empty list>

2
python/testData/psi/FStringTerminatedByQuoteOfStringLiteralInFormatPart.txt
View File

@@ -18,6 +18,8 @@ PyFile:FStringTerminatedByQuoteOfStringLiteralInFormatPart.py
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
PsiErrorElement:End of statement expected
<empty list>

40
python/testSrc/com/jetbrains/python/PythonLexerTest.java
View File

@@ -388,11 +388,41 @@ public class PythonLexerTest extends PyLexerTestCase {
doTest("10_000_000J", "Py:IMAGINARY_LITERAL", "Py:STATEMENT_BREAK");
}
public void testFStringLiterals() {
doTest("s = f'{x}'", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
doTest("s = rf\"{x}\"", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
doTest("s = fr'''{x}\n'''", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
doTest("s = f\"\"\"{x}\n\"\"\"", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
public void testFStringMatchingQuoteRecoveryInsideContentOfNestedStringLiteral() {
doTest("s = f'{ur\"foo'bar\"}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:SINGLE_QUOTED_STRING", "Py:FSTRING_END",
"Py:IDENTIFIER", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testFStringMatchingQuoteRecoveryQuoteOfNestedStringLiteralWithPrefix() {
doTest("s = f'{ur'foo'}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_END",
"Py:IDENTIFIER", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testFStringMatchingQuoteRecoveryQuoteOfNestedStringLiteralWithoutPrefix() {
doTest("s = f'{'foo'}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:FSTRING_END",
"Py:IDENTIFIER", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testNoStatementBreakInsideFragmentOfMultilineFString() {
doTest("s = f'''{1 + \n" +
"2}'''",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:INTEGER_LITERAL", "Py:SPACE", "Py:PLUS", "Py:LINE_BREAK",
"Py:INTEGER_LITERAL", "Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
}
public void testStatementBreakInsideFragmentOfSingleLineFString() {
doTest("s = f'{1 +\n" +
" 2}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:INTEGER_LITERAL", "Py:SPACE", "Py:PLUS", "Py:STATEMENT_BREAK",
"Py:LINE_BREAK", "Py:INDENT", "Py:INTEGER_LITERAL", "Py:RBRACE", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
// PY-21697

16
python/testSrc/com/jetbrains/python/PythonParsingTest.java
View File

@@ -745,6 +745,22 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringBackslashInsteadOfExpression() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringBackslashAfterExpression() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringBackslashBeforeExpression() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringBackslashInsideMultilineExpression() {
doTest(LanguageLevel.PYTHON36);
}
// PY-19036
public void testAwaitInNonAsyncNestedFunction() {
doTest(LanguageLevel.PYTHON35);