MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2025-12-15 02:59:33 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-31442 Lexer handles line breaks in single quoted f-strings

Browse Source
This commit is contained in:
Mikhail Golubev
2018-08-30 21:22:39 +03:00
parent 479b06eaed
commit 99d8ee7bbc
20 changed files with 1226 additions and 774 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1523
python/gen/com/jetbrains/python/lexer/_PythonLexer.java
View File

File diff suppressed because it is too large Load Diff

34
python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt
View File

@@ -66,6 +66,22 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) {
}
}
fun handleLineBreakInFragment(): IElementType {
val text = myLexer.yytext().toString()
// We will return a line break anyway, but we need to transit from FSTRING state of the lexer
findFStringTerminator(text)
return PyTokenTypes.LINE_BREAK
}
fun handleLineBreakInLiteralText(): IElementType {
val text = myLexer.yytext().toString()
val (_, offset) = findFStringTerminator(text)
if (offset == text.length) {
return PyTokenTypes.FSTRING_TEXT
}
return PyTokenTypes.LINE_BREAK
}
fun handleStringLiteral(stringLiteralType: IElementType): IElementType {
val stringText = myLexer.yytext().toString()
val prefixLength = PyStringLiteralUtil.getPrefixLength(stringText)
@@ -86,13 +102,28 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) {
i += 2
continue
}
if (c == '\n') {
val firstSingleQuotedIndex = myFStringStates.indexOfFirst { it.openingQuotes.length == 1 }
if (firstSingleQuotedIndex >= 0) {
if (i == 0) {
myLexer.yybegin(myFStringStates[firstSingleQuotedIndex].oldState)
myFStringStates.subList(firstSingleQuotedIndex, myFStringStates.size).clear()
myLexer.yypushback(text.length - 1)
}
else {
myLexer.yypushback(text.length - i)
}
return Pair(PyTokenTypes.LINE_BREAK, i)
}
}
else {
val nextThree = text.substring(i, Math.min(text.length, i + 3))
for (j in myFStringStates.size - 1 downTo 0) {
val state = myFStringStates[j]
if (nextThree.startsWith(state.openingQuotes)) {
if (i == 0) {
myFStringStates.subList(i, myFStringStates.size).clear()
myLexer.yybegin(state.oldState)
myFStringStates.subList(j, myFStringStates.size).clear()
val unmatched = text.length - state.openingQuotes.length
myLexer.yypushback(unmatched)
}
@@ -102,6 +133,7 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) {
return Pair(PyTokenTypes.FSTRING_END, i)
}
}
}
i++
}
return Pair(null, text.length)

10
python/src/com/jetbrains/python/lexer/Python.flex
View File

@@ -2,10 +2,8 @@
package com.jetbrains.python.lexer;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.containers.ContainerUtil;import com.jetbrains.python.PyTokenTypes;
import com.intellij.util.containers.Stack;
import com.jetbrains.python.PyTokenTypes;
import com.intellij.openapi.util.text.StringUtil;
import com.jetbrains.python.psi.PyStringLiteralUtil;
%%
@@ -104,6 +102,7 @@ return yylength()-s.length();
<FSTRING> {
{FSTRING_TEXT_NO_QUOTES} { return PyTokenTypes.FSTRING_TEXT; }
[\n] { return fStringHelper.handleLineBreakInLiteralText(); }
{FSTRING_QUOTES} { return fStringHelper.handleFStringEnd(); }
"{" { return fStringHelper.handleFragmentStart(); }
}
@@ -125,16 +124,19 @@ return yylength()-s.length();
{SINGLE_QUOTED_STRING} { return fStringHelper.handleStringLiteral(PyTokenTypes.SINGLE_QUOTED_STRING); }
{TRIPLE_QUOTED_LITERAL} { return fStringHelper.handleStringLiteral(PyTokenTypes.TRIPLE_QUOTED_STRING); }
[\n] { return fStringHelper.handleLineBreakInFragment(); }
// Should be impossible inside expression fragments: any openingQuotes should be matched as a string literal there
// {FSTRING_QUOTES} { return hasMatchingFStringStart(yytext().toString()) ? PyTokenTypes.FSTRING_END : PyTokenTypes.FSTRING_TEXT; }
}
<FSTRING_FRAGMENT_FORMAT> {
{FSTRING_FORMAT_TEXT_NO_QUOTES} { return PyTokenTypes.FSTRING_TEXT; }
[\n] { return fStringHelper.handleLineBreakInLiteralText(); }
"{" { return fStringHelper.handleFragmentStart(); }
"}" { return fStringHelper.handleFragmentEnd(); }
// format part of a fragment can contain openingQuotes
// format part of a fragment can contain opening quotes
{FSTRING_QUOTES} { return fStringHelper.handleFStringEnd(); }
}

2
python/testData/psi/FStringTerminatedByEscapedLineBreakInNestedStringLiteral.py Normal file
View File

@@ -0,0 +1,2 @@
s = f"{f'''{'\
'}'''}"

26
python/testData/psi/FStringTerminatedByEscapedLineBreakInNestedStringLiteral.txt Normal file
View File

@@ -0,0 +1,26 @@
PyFile:FStringTerminatedByEscapedLineBreakInNestedStringLiteral.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {f'''{'\
'}'''}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: {'\
'}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'''')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression:
PsiElement(Py:SINGLE_QUOTED_STRING)(''\\n'')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''''')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)('"')

33
python/testData/psi/FStringTerminatedByLineBreakInExpression.txt Normal file
View File

@@ -0,0 +1,33 @@
PyFile:FStringTerminatedByLineBreakInExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {1 +
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyBinaryExpression
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(Py:PLUS)('+')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PyExpressionStatement
PyStringLiteralExpression:
PsiElement(Py:SINGLE_QUOTED_STRING)(''')

42
python/testData/psi/FStringTerminatedByLineBreakInExpressionInFormatPart.txt Normal file
View File

@@ -0,0 +1,42 @@
PyFile:FStringTerminatedByLineBreakInExpressionInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {42:{1 +
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyBinaryExpression
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(Py:PLUS)('+')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PyExpressionStatement
PyStringLiteralExpression:
PsiElement(Py:SINGLE_QUOTED_STRING)(''')

34
python/testData/psi/FStringTerminatedByLineBreakInFormatPart.txt Normal file
View File

@@ -0,0 +1,34 @@
PyFile:FStringTerminatedByLineBreakInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo{42:bar
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_TEXT)('bar')
PsiErrorElement:} expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyReferenceExpression: baz
PsiElement(Py:IDENTIFIER)('baz')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PyExpressionStatement
PyStringLiteralExpression:
PsiElement(Py:SINGLE_QUOTED_STRING)(''')

22
python/testData/psi/FStringTerminatedByLineBreakInLiteralPart.txt Normal file
View File

@@ -0,0 +1,22 @@
PyFile:FStringTerminatedByLineBreakInLiteralPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyReferenceExpression: bar
PsiElement(Py:IDENTIFIER)('bar')
PsiErrorElement:End of statement expected
<empty list>
PyExpressionStatement
PyStringLiteralExpression:
PsiElement(Py:SINGLE_QUOTED_STRING)(''')

40
python/testData/psi/FStringTerminatedByLineBreakInNestedExpression.txt Normal file
View File

@@ -0,0 +1,40 @@
PyFile:FStringTerminatedByLineBreakInNestedExpression.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {f'{1 +
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: {1 +
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyBinaryExpression
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(Py:PLUS)('+')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PyExpressionStatement
PyStringLiteralExpression: }"
PsiElement(Py:SINGLE_QUOTED_STRING)(''}"')

49
python/testData/psi/FStringTerminatedByLineBreakInNestedExpressionInFormatPart.txt Normal file
View File

@@ -0,0 +1,49 @@
PyFile:FStringTerminatedByLineBreakInNestedExpressionInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {f'{42:{1 +
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: {42:{1 +
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyBinaryExpression
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(Py:PLUS)('+')
PsiErrorElement:expression expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PyExpressionStatement
PyStringLiteralExpression: }"
PsiElement(Py:SINGLE_QUOTED_STRING)(''}"')

41
python/testData/psi/FStringTerminatedByLineBreakInNestedFormatPart.txt Normal file
View File

@@ -0,0 +1,41 @@
PyFile:FStringTerminatedByLineBreakInNestedFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {f'foo{42:bar
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: foo{42:bar
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_TEXT)('bar')
PsiErrorElement:} expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyReferenceExpression: baz
PsiElement(Py:IDENTIFIER)('baz')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACE)('}')
PsiErrorElement:Statement expected, found Py:RBRACE
<empty list>
PyExpressionStatement
PyStringLiteralExpression: }"
PsiElement(Py:SINGLE_QUOTED_STRING)(''}"')

31
python/testData/psi/FStringTerminatedByLineBreakInNestedLiteralPart.txt Normal file
View File

@@ -0,0 +1,31 @@
PyFile:FStringTerminatedByLineBreakInNestedLiteralPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {f'foo
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: foo
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PsiErrorElement:Unexpected f-string token
<empty list>
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyReferenceExpression: bar
PsiElement(Py:IDENTIFIER)('bar')
PsiErrorElement:End of statement expected
<empty list>
PyExpressionStatement
PyStringLiteralExpression: }"
PsiElement(Py:SINGLE_QUOTED_STRING)(''}"')

2
python/testData/psi/FStringTerminatedByLineBreakInNestedStringLiteral.py
View File

@@ -1,2 +0,0 @@
s = f"{f'{"""
"""}'}"

2
python/testData/psi/FStringTerminatedByLineBreakInNestedStringLiteralInFormatPart.py Normal file
View File

@@ -0,0 +1,2 @@
s = f"{f'''{foo:{'\
'}}'''}"

33
python/testData/psi/FStringTerminatedByLineBreakInNestedStringLiteralInFormatPart.txt Normal file
View File

@@ -0,0 +1,33 @@
PyFile:FStringTerminatedByLineBreakInNestedStringLiteralInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {f'''{foo:{'\
'}}'''}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f"')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression: {foo:{'\
'}}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'''')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: foo
PsiElement(Py:IDENTIFIER)('foo')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression:
PsiElement(Py:SINGLE_QUOTED_STRING)(''\\n'')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''''')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)('"')

22
python/testData/psi/FStringTerminatedByLineBreakInStringLiteral.txt Normal file
View File

@@ -0,0 +1,22 @@
PyFile:FStringTerminatedByLineBreakInStringLiteral.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {"""
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression:
PsiElement(Py:TRIPLE_QUOTED_STRING)('"""')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyStringLiteralExpression: }'
PsiElement(Py:TRIPLE_QUOTED_STRING)('"""}'')

28
python/testData/psi/FStringTerminatedByLineBreakInStringLiteralInFormatPart.txt Normal file
View File

@@ -0,0 +1,28 @@
PyFile:FStringTerminatedByLineBreakInStringLiteralInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {foo:{"""
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: foo
PsiElement(Py:IDENTIFIER)('foo')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyStringLiteralExpression:
PsiElement(Py:TRIPLE_QUOTED_STRING)('"""')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:Unexpected f-string token
<empty list>
PsiWhiteSpace('\n')
PyExpressionStatement
PyStringLiteralExpression: }}'
PsiElement(Py:TRIPLE_QUOTED_STRING)('"""}}'')

2
python/testData/psi/FStringTerminatedByLineBreakInStringLiteralInNestedFormatPart.py
View File

@@ -1,2 +0,0 @@
s = f"{f'{foo:{"""
"""}}'}"

4
python/testSrc/com/jetbrains/python/PythonParsingTest.java
View File

@@ -713,7 +713,9 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringTerminatedByLineBreakInNestedStringLiteral() {
// not possible to come up with such case without escaping: triple-quoted string inside
// two nested f-strings with different types of quotes
public void testFStringTerminatedByEscapedLineBreakInNestedStringLiteral() {
doTest(LanguageLevel.PYTHON36);
}