MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-02-05 08:06:56 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-31442 Handles escape sequences in format parts and backslashes before curly braces

Browse Source
This commit is contained in:
Mikhail Golubev
2018-09-04 21:13:24 +03:00
parent 88119d9c59
commit 2287a27ed0
16 changed files with 712 additions and 533 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1070
python/gen/com/jetbrains/python/lexer/_PythonLexer.java
View File

File diff suppressed because it is too large Load Diff

8
python/src/com/jetbrains/python/lexer/Python.flex
View File

@@ -72,11 +72,13 @@ FSTRING_PREFIX = [UuBbCcRr]{0,3}[fF][UuBbCcRr]{0,3}
FSTRING_START = {FSTRING_PREFIX} (\"\"\"|'''|\"|')
FSTRING_QUOTES = (\"{1,3}|'{1,3})
FSTRING_ESCAPED_LBRACE = "{{"
FSTRING_ESCAPE_SEQUENCE = \\[^{}\r\n]
BACKSLASH_ESCAPED_BRACES = "\\{" | "\\}"
// TODO report it in annotation
//FSTRING_ESCAPED_RBRACE = "}}"
NAMED_UNICODE_ESCAPE = \\N\{[\w ]*\}?
FSTRING_TEXT_NO_QUOTES = ([^\\\'\"\r\n{] | {NAMED_UNICODE_ESCAPE} | {ESCAPE_SEQUENCE} | {FSTRING_ESCAPED_LBRACE} | (\\[\r\n]))+
FSTRING_FORMAT_TEXT_NO_QUOTES = ([^\\\'\"\r\n{}] | {NAMED_UNICODE_ESCAPE} | (\\[\r\n]))+
FSTRING_TEXT_NO_QUOTES = ([^\\\'\"\r\n{] | {NAMED_UNICODE_ESCAPE} | {FSTRING_ESCAPE_SEQUENCE} | {FSTRING_ESCAPED_LBRACE} | (\\[\r\n]) )+
FSTRING_FORMAT_TEXT_NO_QUOTES = ([^\\\'\"\r\n{}] | {NAMED_UNICODE_ESCAPE} | {FSTRING_ESCAPE_SEQUENCE} | (\\[\r\n]) )+
FSTRING_FRAGMENT_TYPE_CONVERSION = "!" [^=:'\"} \t\r\n]*
%state PENDING_DOCSTRING
@@ -102,6 +104,7 @@ return yylength()-s.length();
<FSTRING> {
{FSTRING_TEXT_NO_QUOTES} { return PyTokenTypes.FSTRING_TEXT; }
[\n] { return fStringHelper.handleLineBreakInLiteralText(); }
{BACKSLASH_ESCAPED_BRACES} { yypushback(1); return PyTokenTypes.FSTRING_TEXT; }
{FSTRING_QUOTES} { return fStringHelper.handleFStringEnd(); }
"{" { return fStringHelper.handleFragmentStart(); }
}
@@ -132,6 +135,7 @@ return yylength()-s.length();
<FSTRING_FRAGMENT_FORMAT> {
{FSTRING_FORMAT_TEXT_NO_QUOTES} { return PyTokenTypes.FSTRING_TEXT; }
[\n] { return fStringHelper.handleLineBreakInLiteralText(); }
{BACKSLASH_ESCAPED_BRACES} { yypushback(1); return PyTokenTypes.FSTRING_TEXT; }
"{" { return fStringHelper.handleFragmentStart(); }
"}" { return fStringHelper.handleFragmentEnd(); }

1
python/testData/psi/FStringEscapeSequenceInFormatPart.py Normal file
View File

@@ -0,0 +1 @@
s = f'{42:\\\n}'

19
python/testData/psi/FStringEscapeSequenceInFormatPart.txt Normal file
View File

@@ -0,0 +1,19 @@
PyFile:FStringEscapeSequenceInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {42:\\\n}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_TEXT)('\\\n')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')

1
python/testData/psi/FStringEscapeSequenceInLiteralPart.py Normal file
View File

@@ -0,0 +1 @@
s = f'\n{42}\\'

19
python/testData/psi/FStringEscapeSequenceInLiteralPart.txt Normal file
View File

@@ -0,0 +1,19 @@
PyFile:FStringEscapeSequenceInLiteralPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
{42}\
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('\n')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('\\')
PsiElement(Py:FSTRING_END)(''')

2
python/testData/psi/FStringEscapedLineBreakInLiteralPart.py Normal file
View File

@@ -0,0 +1,2 @@
s = f'foo\
bar{42}'

18
python/testData/psi/FStringEscapedLineBreakInLiteralPart.txt Normal file
View File

@@ -0,0 +1,18 @@
PyFile:FStringEscapedLineBreakInLiteralPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo
bar{42}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo\\nbar')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')

1
python/testData/psi/FStringEscapedSlashBeforeClosingQuoteInFormatPart.py Normal file
View File

@@ -0,0 +1 @@
s = f'{42:\\'

20
python/testData/psi/FStringEscapedSlashBeforeClosingQuoteInFormatPart.txt Normal file
View File

@@ -0,0 +1,20 @@
PyFile:FStringEscapedSlashBeforeClosingQuoteInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {42:\\
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_TEXT)('\\')
PsiErrorElement:} expected
<empty list>
PsiElement(Py:FSTRING_END)(''')

1
python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.py Normal file
View File

@@ -0,0 +1 @@
s = f'foo\{42}'

18
python/testData/psi/FStringSingleSlashBeforeLeftBraceInLiteralPart.txt Normal file
View File

@@ -0,0 +1,18 @@
PyFile:FStringSingleSlashBeforeLeftBraceInLiteralPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: foo\{42}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PsiElement(Py:FSTRING_TEXT)('foo')
PsiElement(Py:FSTRING_TEXT)('\')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')

1
python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.py Normal file
View File

@@ -0,0 +1 @@
s = f'{42:foo\{bar}baz\}'

27
python/testData/psi/FStringSingleSlashesBeforeBracesInFormatPart.txt Normal file
View File

@@ -0,0 +1,27 @@
PyFile:FStringSingleSlashesBeforeBracesInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {42:foo\{bar}baz\}
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_TEXT)('foo')
PsiElement(Py:FSTRING_TEXT)('\')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: bar
PsiElement(Py:IDENTIFIER)('bar')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('baz')
PsiElement(Py:FSTRING_TEXT)('\')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')

15
python/testSrc/com/jetbrains/python/PythonLexerTest.java
View File

@@ -450,6 +450,21 @@ public class PythonLexerTest extends PyLexerTestCase {
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
}
public void testFStringBackslashEscapedBraces() {
doTest("s = f'foo\\{x}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END",
"Py:FSTRING_END", "Py:STATEMENT_BREAK");
doTest("s = f'{x:foo\\{y}bar\\}'",
"Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE",
"Py:FSTRING_START", "Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER",
"Py:FSTRING_FRAGMENT_FORMAT_START", "Py:FSTRING_TEXT", "Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_START", "Py:IDENTIFIER", "Py:FSTRING_FRAGMENT_END",
"Py:FSTRING_TEXT", "Py:FSTRING_TEXT",
"Py:FSTRING_FRAGMENT_END", "Py:FSTRING_END", "Py:STATEMENT_BREAK");
}
// PY-21697
public void testTripleSingleQuotedStringWithEscapedSlashAfterOneQuote() {
doTest("s = '''\n" +

24
python/testSrc/com/jetbrains/python/PythonParsingTest.java
View File

@@ -793,6 +793,30 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringEscapeSequenceInLiteralPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringEscapeSequenceInFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringEscapedSlashBeforeClosingQuoteInFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringSingleSlashBeforeLeftBraceInLiteralPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringSingleSlashesBeforeBracesInFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringEscapedLineBreakInLiteralPart() {
doTest(LanguageLevel.PYTHON36);
}
// PY-19036
public void testAwaitInNonAsyncNestedFunction() {
doTest(LanguageLevel.PYTHON35);