MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-18 09:34:34 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-31442 Parse type conversions (!r/!s/!a) inside fragments

Browse Source 
Also, improved handling of incomplete fragments inside f-strings.
This commit is contained in:
Mikhail Golubev
2018-10-01 12:46:42 +03:00
parent 40c1e8b949
commit 0300a411d6
27 changed files with 1059 additions and 749 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/gen/com/jetbrains/python/lexer/_PythonLexer.java
+772 -747
View File
File diff suppressed because it is too large Load Diff
python/psi-api/src/com/jetbrains/python/PyTokenTypes.java
+1
View File
@@ -185,4 +185,5 @@ public class PyTokenTypes {
public static final PyElementType FSTRING_FRAGMENT_START = new PyElementType("FSTRING_FRAGMENT_START");
public static final PyElementType FSTRING_FRAGMENT_END = new PyElementType("FSTRING_FRAGMENT_END");
public static final PyElementType FSTRING_FRAGMENT_FORMAT_START = new PyElementType("FSTRING_FRAGMENT_FORMAT_START");
public static final PyElementType FSTRING_FRAGMENT_TYPE_CONVERSION = new PyElementType("FSTRING_FRAGMENT_TYPE_CONVERSION");
}
python/src/com/jetbrains/python/lexer/Python.flex
+7
View File
@@ -81,6 +81,7 @@ FSTRING_ESCAPED_LBRACE = "{{"
//FSTRING_ESCAPED_RBRACE = "}}"
FSTRING_TEXT = ([^\\\'\r\n{] | {ESCAPE_SEQUENCE} | {FSTRING_ESCAPED_LBRACE} | (\\[\r\n]))+
FSTRING_FORMAT_TEXT = ([^\\\'\r\n{}] | (\\[\r\n]))+
FSTRING_FRAGMENT_TYPE_CONVERSION = "!" [^=:'\"} \t\r\n]*
%state PENDING_DOCSTRING
%state IN_DOCSTRING_OWNER
@@ -171,8 +172,12 @@ return yylength()-s.length();
"}" { if (braceBalance == 0) { popFStringFragment(); return PyTokenTypes.FSTRING_FRAGMENT_END; }
else { braceBalance--; return PyTokenTypes.RBRACE; } }
{FSTRING_FRAGMENT_TYPE_CONVERSION} { return PyTokenTypes.FSTRING_FRAGMENT_TYPE_CONVERSION; }
":" { if (braceBalance == 0) { yybegin(FSTRING_FRAGMENT_FORMAT); return PyTokenTypes.FSTRING_FRAGMENT_FORMAT_START; }
else { return PyTokenTypes.COLON; } }
{FSTRING_END} { popFStringFragment(); popFString(); return PyTokenTypes.FSTRING_END; }
}
<FSTRING_FRAGMENT_FORMAT> {
@@ -180,6 +185,8 @@ return yylength()-s.length();
"{" { pushFStringFragment(); return PyTokenTypes.FSTRING_FRAGMENT_START; }
"}" { if (braceBalance == 0) { popFStringFragment(); return PyTokenTypes.FSTRING_FRAGMENT_END; }
else { braceBalance--; return PyTokenTypes.RBRACE; } }
{FSTRING_END} { popFStringFragment(); popFString(); return PyTokenTypes.FSTRING_END; }
}
<YYINITIAL> {
python/src/com/jetbrains/python/parsing/ExpressionParsing.java
+11 -2
View File
@@ -146,10 +146,19 @@ public class ExpressionParsing extends Parsing {
final PsiBuilder.Marker marker = builder.mark();
nextToken();
myContext.getExpressionParser().parseExpression();
if (atToken(PyTokenTypes.FSTRING_FRAGMENT_FORMAT_START)) {
final boolean hasTypeConversion = matchToken(PyTokenTypes.FSTRING_FRAGMENT_TYPE_CONVERSION);
final boolean hasFormatPart = atToken(PyTokenTypes.FSTRING_FRAGMENT_FORMAT_START);
if (hasFormatPart) {
parseFStringFragmentFormatPart();
}
checkMatches(PyTokenTypes.FSTRING_FRAGMENT_END, "} expected");
String errorMessage = "} expected";
if (!hasFormatPart) {
errorMessage = ": or " + errorMessage;
if (!hasTypeConversion) {
errorMessage = "type conversion, " + errorMessage;
}
}
checkMatches(PyTokenTypes.FSTRING_FRAGMENT_END, errorMessage);
marker.done(PyElementTypes.FSTRING_FRAGMENT);
}
}
python/testData/psi/FStringFragmentDuplicateTypeConversion.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!r!r}'
python/testData/psi/FStringFragmentDuplicateTypeConversion.txt
+17
View File
@@ -0,0 +1,17 @@
PyFile:FStringFragmentDuplicateTypeConversion.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!r!r')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringFragmentIllegalTypeConversion.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!!}'
python/testData/psi/FStringFragmentIllegalTypeConversion.txt
+17
View File
@@ -0,0 +1,17 @@
PyFile:FStringFragmentIllegalTypeConversion.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!!')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringFragmentIncompleteTypeConversionBeforeClosingBrace.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!}'
python/testData/psi/FStringFragmentIncompleteTypeConversionBeforeClosingBrace.txt
+17
View File
@@ -0,0 +1,17 @@
PyFile:FStringFragmentIncompleteTypeConversionBeforeClosingBrace.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringFragmentIncompleteTypeConversionBeforeClosingQuote.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!'
python/testData/psi/FStringFragmentIncompleteTypeConversionBeforeClosingQuote.txt
+18
View File
@@ -0,0 +1,18 @@
PyFile:FStringFragmentIncompleteTypeConversionBeforeClosingQuote.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!')
PsiErrorElement:: or } expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringFragmentIncompleteTypeConversionBeforeColon.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!:}'
python/testData/psi/FStringFragmentIncompleteTypeConversionBeforeColon.txt
+19
View File
@@ -0,0 +1,19 @@
PyFile:FStringFragmentIncompleteTypeConversionBeforeColon.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringFragmentTypeConversion.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!r}'
python/testData/psi/FStringFragmentTypeConversion.txt
+17
View File
@@ -0,0 +1,17 @@
PyFile:FStringFragmentTypeConversion.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!r')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringFragmentTypeConversionAfterFormatPart.py
+1
View File
@@ -0,0 +1 @@
s = f'{42:{width}!r}'
python/testData/psi/FStringFragmentTypeConversionAfterFormatPart.txt
+24
View File
@@ -0,0 +1,24 @@
PyFile:FStringFragmentTypeConversionAfterFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: width
PsiElement(Py:IDENTIFIER)('width')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('!r')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringIncompleteFragment.py
+1
View File
@@ -0,0 +1 @@
s = f'{42'
python/testData/psi/FStringIncompleteFragment.txt
+17
View File
@@ -0,0 +1,17 @@
PyFile:FStringIncompleteFragment.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringIncompleteFragmentWithFormatPart.py
+1
View File
@@ -0,0 +1 @@
s = f'{42:{width}'
python/testData/psi/FStringIncompleteFragmentWithFormatPart.txt
+24
View File
@@ -0,0 +1,24 @@
PyFile:FStringIncompleteFragmentWithFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: width
PsiElement(Py:IDENTIFIER)('width')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiErrorElement:} expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringIncompleteFragmentWithTypeConversion.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!r'
python/testData/psi/FStringIncompleteFragmentWithTypeConversion.txt
+18
View File
@@ -0,0 +1,18 @@
PyFile:FStringIncompleteFragmentWithTypeConversion.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!r')
PsiErrorElement:: or } expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringIncompleteFragmentWithTypeConversionAndFormatPart.py
+1
View File
@@ -0,0 +1 @@
s = f'{42!r:{width}'
python/testData/psi/FStringIncompleteFragmentWithTypeConversionAndFormatPart.txt
+25
View File
@@ -0,0 +1,25 @@
PyFile:FStringIncompleteFragmentWithTypeConversionAndFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression:
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:FSTRING_FRAGMENT_TYPE_CONVERSION)('!r')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyReferenceExpression: width
PsiElement(Py:IDENTIFIER)('width')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiErrorElement:} expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
python/testSrc/com/jetbrains/python/PythonParsingTest.java
+44
View File
@@ -553,6 +553,50 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentTypeConversion() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentIllegalTypeConversion() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentIncompleteTypeConversionBeforeColon() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentIncompleteTypeConversionBeforeClosingBrace() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentIncompleteTypeConversionBeforeClosingQuote() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentTypeConversionAfterFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringFragmentDuplicateTypeConversion() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringIncompleteFragment() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringIncompleteFragmentWithTypeConversion() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringIncompleteFragmentWithFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringIncompleteFragmentWithTypeConversionAndFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
// PY-19036
public void testAwaitInNonAsyncNestedFunction() {
doTest(LanguageLevel.PYTHON35);