MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-31442 Include comments and whitespaces inside fragments

Browse Source 
It's beneficial for two reasons: it allows us to avoid occasional
PsiWhitespaces at the end of incomplete fragments and also we
can now better report illegal line comments this way, because even though
they still terminate f-strings they are now under the corresponding AST
nodes for expression fragments and can be processed by the annotator
in the same fashion.
This commit is contained in:
Mikhail Golubev
2018-10-01 12:46:45 +03:00
parent 2f9b1c62ff
commit b09241cb09
11 changed files with 53 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/parsing/ExpressionParsing.java
+4
View File
@@ -16,6 +16,7 @@
package com.jetbrains.python.parsing;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.WhitespacesAndCommentsBinder;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
@@ -30,6 +31,7 @@ import static com.jetbrains.python.PyBundle.message;
*/
public class ExpressionParsing extends Parsing {
private static final Logger LOG = Logger.getInstance("#ru.yole.pythonlanguage.parsing.ExpressionParsing");
public static final WhitespacesAndCommentsBinder CONSUME_COMMENTS_AND_SPACES_TO_LEFT = (tokens, atStreamEdge, getter) -> tokens.size();
public ExpressionParsing(ParsingContext context) {
super(context);
@@ -176,6 +178,7 @@ public class ExpressionParsing extends Parsing {
}
if (recovery) {
recoveryMarker.error(parsedExpression ? "unexpected expression part" : "expression expected");
recoveryMarker.setCustomEdgeTokenBinders(null, CONSUME_COMMENTS_AND_SPACES_TO_LEFT);
}
else {
recoveryMarker.drop();
@@ -193,6 +196,7 @@ public class ExpressionParsing extends Parsing {
}
}
checkMatches(PyTokenTypes.FSTRING_FRAGMENT_END, errorMessage);
marker.setCustomEdgeTokenBinders(null, CONSUME_COMMENTS_AND_SPACES_TO_LEFT);
marker.done(PyElementTypes.FSTRING_FRAGMENT);
}
}
python/testData/highlighting/fStringEmptyExpressions.py
+3 -3
View File
@@ -4,7 +4,7 @@ f'{<EOLError descr="type conversion, : or } expected"></EOLError><EOLError descr
f'{<error descr="expression expected">!</error>r}'
f'{<error descr="expression expected">:</error>2.3}'
f'{42:2.{<error descr="expression expected">}</error>}'
f'{<error descr="expression expected"> </error> }'
f'{<error descr="expression expected"> </error>}'
f'{42:{<error descr="expression expected"> </error>}}'
f'{<error descr="expression expected"> </error> :{<error descr="expression expected"><error descr="type conversion, : or } expected"> </error></error> '
f'{<error descr="expression expected"> </error> !r:{<error descr="expression expected"> </error> :42}}'
f'{<error descr="expression expected"> </error>:{<error descr="expression expected"> </error><error descr="type conversion, : or } expected">'</error>
f'{<error descr="expression expected"> </error>!r:{<error descr="expression expected"> </error>:42}}'
python/testData/highlighting/fStringHashSigns.py
+6 -6
View File
@@ -1,9 +1,9 @@
f'{<error descr="' expected"><error descr="expression expected"><error descr="type conversion, : or } expected">#</error></error></error>'
f'{<error descr="' expected"><error descr="expression expected"><error descr="type conversion, : or } expected">#</error></error></error>
f'{<error descr="' expected"><error descr="expression expected"><error descr="type conversion, : or } expected">#</error></error></error>foo#}'
f'{42:<error descr="' expected"><error descr="} expected">#</error></error>}'
f'{42:{<error descr="' expected"><error descr="expression expected"><error descr="type conversion, : or } expected">#</error></error></error>}}'
f'{x<error descr="' expected"><error descr="type conversion, : or } expected"> </error></error>### foo}'
f'{<error descr="Expression fragments inside f-strings cannot include line comments"><error descr="expression expected">#'</error></error><EOLError descr="type conversion, : or } expected"></EOLError><EOLError descr="' expected"></EOLError>
f'{<error descr="Expression fragments inside f-strings cannot include line comments"><error descr="expression expected">#</error></error><EOLError descr="type conversion, : or } expected"></EOLError><EOLError descr="' expected"></EOLError>
f'{<error descr="Expression fragments inside f-strings cannot include line comments"><error descr="expression expected">#foo#}'</error></error><EOLError descr="type conversion, : or } expected"></EOLError><EOLError descr="' expected"></EOLError>
f'{42:<error descr="Expression fragments inside f-strings cannot include line comments"><error descr="} expected">#</error>}'</error><EOLError descr="' expected"></EOLError>
f'{42:{<error descr="Expression fragments inside f-strings cannot include line comments"><error descr="expression expected">#}}'</error></error><EOLError descr="type conversion, : or } expected"></EOLError><EOLError descr="' expected"></EOLError>
f'{x<error descr="type conversion, : or } expected"> </error><error descr="Expression fragments inside f-strings cannot include line comments">### foo}'</error><EOLError descr="' expected"></EOLError>
f'{"###"}'
f'''{[
42 <error descr="Expression fragments inside f-strings cannot include line comments"># foo</error>
python/testData/highlighting/fStringTooDeeplyNestedExpressionFragments.py
+1 -1
View File
@@ -1,5 +1,5 @@
f'{x:{y:<error descr="Expression fragment inside f-string is nested too deeply">{<error descr="expression expected">}</error></error>}}'
f'{x:{y:<error descr="Expression fragment inside f-string is nested too deeply">{</error><error descr="' expected"><error descr="expression expected"><error descr="type conversion, : or } expected">#</error></error></error> foo}}}'
f'{x:{y:<error descr="Expression fragment inside f-string is nested too deeply">{<error descr="Expression fragments inside f-strings cannot include line comments"><error descr="expression expected"># foo}}}'</error></error></error><EOLError descr="type conversion, : or } expected"></EOLError><EOLError descr="' expected"></EOLError>
f'{x:{y:<error descr="Expression fragment inside f-string is nested too deeply">{z<error descr="Illegal conversion character 'z': should be one of 's', 'r', 'a'">!z</error>}</error>}}'
f'{x:{y:<error descr="Expression fragment inside f-string is nested too deeply">{z:<error descr="Expression fragment inside f-string is nested too deeply">{42}</error>}</error>}}'
f'{<error descr="expression expected">:</error>{<error descr="expression expected">:</error><error descr="Expression fragment inside f-string is nested too deeply">{<error descr="expression expected">:</error><error descr="Expression fragment inside f-string is nested too deeply">{<error descr="expression expected">}</error></error>}</error>}}'
python/testData/psi/FStringBackslashAfterExpression.txt
+1 -1
View File
@@ -15,7 +15,7 @@ PyFile:FStringBackslashAfterExpression.py
PsiElement(Py:INTEGER_LITERAL)('42')
PsiErrorElement:unexpected expression part
PsiElement(Py:BACKSLASH)(' \')
PsiWhiteSpace(' ')
PsiWhiteSpace(' ')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_TEXT)('bar')
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/FStringTrailingWhitespaceInIncompleteFragmentInFormatPart.py
+1
View File
@@ -0,0 +1 @@
s = f'{42:{ '
python/testData/psi/FStringTrailingWhitespaceInIncompleteFragmentInFormatPart.txt
+23
View File
@@ -0,0 +1,23 @@
PyFile:FStringTrailingWhitespaceInIncompleteFragmentInFormatPart.py
PyAssignmentStatement
PyTargetExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {42:{
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PyFStringFragmentFormatPart
PsiElement(Py:FSTRING_FRAGMENT_FORMAT_START)(':')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
PsiWhiteSpace(' ')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiElement(Py:FSTRING_END)(''')
python/testData/psi/MultilineFStringContainsCommentInsteadOfExpression.txt
+3 -4
View File
@@ -13,9 +13,8 @@ PyFile:MultilineFStringContainsCommentInsteadOfExpression.py
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# comment')
PsiWhiteSpace('\n')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# comment')
PsiWhiteSpace('\n')
PsiElement(Py:FSTRING_FRAGMENT_END)('}')
PsiElement(Py:FSTRING_END)(''''')
python/testData/psi/SingleLineFStringContainsCommentInsideExpression.txt
+4 -4
View File
@@ -5,7 +5,7 @@ PyFile:SingleLineFStringContainsCommentInsideExpression.py
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {42
PyStringLiteralExpression: {42 # comment}'
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
@@ -14,7 +14,7 @@ PyFile:SingleLineFStringContainsCommentInsideExpression.py
PsiElement(Py:INTEGER_LITERAL)('42')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiWhiteSpace(' ')
PsiComment(Py:END_OF_LINE_COMMENT)('# comment}'')
PsiErrorElement:' expected
<empty list>
PsiWhiteSpace(' ')
PsiComment(Py:END_OF_LINE_COMMENT)('# comment}'')
<empty list>
python/testData/psi/SingleLineFStringContainsCommentInsteadOfExpression.txt
+3 -4
View File
@@ -5,15 +5,14 @@ PyFile:SingleLineFStringContainsCommentInsteadOfExpression.py
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PyStringLiteralExpression: {
PyStringLiteralExpression: {# comment}'
PyFormattedStringNode
PsiElement(Py:FSTRING_START)('f'')
PyFStringFragment
PsiElement(Py:FSTRING_FRAGMENT_START)('{')
PsiErrorElement:expression expected
<empty list>
PsiComment(Py:END_OF_LINE_COMMENT)('# comment}'')
PsiErrorElement:type conversion, : or } expected
<empty list>
PsiErrorElement:' expected
<empty list>
PsiComment(Py:END_OF_LINE_COMMENT)('# comment}'')
<empty list>
python/testSrc/com/jetbrains/python/PythonParsingTest.java
+4
View File
@@ -825,6 +825,10 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON36);
}
public void testFStringTrailingWhitespaceInIncompleteFragmentInFormatPart() {
doTest(LanguageLevel.PYTHON36);
}
// PY-19036
public void testAwaitInNonAsyncNestedFunction() {
doTest(LanguageLevel.PYTHON35);