MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Fixed parsing ellipsis in slice lists (PY-8606)

Browse Source
This commit is contained in:
Andrey Vlasovskikh
2013-01-28 20:49:48 +04:00
parent 1c00750e19
commit 8e4270e44a
8 changed files with 114 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/parsing/ExpressionParsing.java
+35 -36
View File
@@ -97,7 +97,7 @@ public class ExpressionParsing extends Parsing {
expr.done(PyElementTypes.LIST_LITERAL_EXPRESSION);
return;
}
if (!parseSingleExpression(isTargetExpression)) {
if (!parseSingleExpression(isTargetExpression, false)) {
builder.error(message("PARSE.expected.expression"));
}
if (builder.getTokenType() == PyTokenTypes.FOR_KEYWORD) {
@@ -111,7 +111,7 @@ public class ExpressionParsing extends Parsing {
if (atToken(PyTokenTypes.RBRACKET)) {
break;
}
if (!parseSingleExpression(isTargetExpression)) {
if (!parseSingleExpression(isTargetExpression, false)) {
builder.error(message("PARSE.expected.expr.or.comma.or.bracket"));
break;
}
@@ -154,7 +154,7 @@ public class ExpressionParsing extends Parsing {
result = parseORTestExpression(false, false);
}
else {
result = parseTupleExpression(false, false, true);
result = parseTupleExpression(false, false, true, false);
}
if (!result) {
myBuilder.error("expression expected");
@@ -172,7 +172,7 @@ public class ExpressionParsing extends Parsing {
}
final PsiBuilder.Marker firstExprMarker = myBuilder.mark();
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
myBuilder.error("expression expected");
firstExprMarker.drop();
expr.done(PyElementTypes.DICT_LITERAL_EXPRESSION);
@@ -198,7 +198,7 @@ public class ExpressionParsing extends Parsing {
}
private void parseDictLiteralTail(PsiBuilder.Marker startMarker, PsiBuilder.Marker firstKeyValueMarker) {
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
myBuilder.error("expression expected");
firstKeyValueMarker.done(PyElementTypes.KEY_VALUE_EXPRESSION);
if (atToken(PyTokenTypes.RBRACE)) {
@@ -225,12 +225,12 @@ public class ExpressionParsing extends Parsing {
private boolean parseKeyValueExpression() {
final PsiBuilder.Marker marker = myBuilder.mark();
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
marker.drop();
return false;
}
checkMatches(PyTokenTypes.COLON, message("PARSE.expected.colon"));
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
myBuilder.error("value expression expected");
marker.drop();
return false;
@@ -242,7 +242,7 @@ public class ExpressionParsing extends Parsing {
private void parseSetLiteralTail(PsiBuilder.Marker startMarker) {
while (myBuilder.getTokenType() != PyTokenTypes.RBRACE) {
checkMatches(PyTokenTypes.COMMA, message("PARSE.expected.comma"));
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
break;
}
}
@@ -325,9 +325,7 @@ public class ExpressionParsing extends Parsing {
parseSliceEnd(expr, sliceItemStart);
}
else {
if (!parseEllipsis()) {
parseSingleExpression(false);
}
parseSingleExpression(false, true);
if (atToken(PyTokenTypes.COLON)) {
sliceOrTupleStart.drop();
parseSliceEnd(expr, sliceItemStart);
@@ -336,7 +334,7 @@ public class ExpressionParsing extends Parsing {
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
if (!parseSliceListTail(expr, sliceOrTupleStart)) {
sliceOrTupleStart.rollbackTo();
if (!parseTupleExpression(false, false, false)) {
if (!parseTupleExpression(false, false, false, true)) {
myBuilder.error("tuple expression expected");
}
checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket"));
@@ -403,13 +401,13 @@ public class ExpressionParsing extends Parsing {
sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION);
}
else {
parseSingleExpression(false);
parseSingleExpression(false, true);
}
if (!BRACKET_COLON_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error(message("PARSE.expected.colon.or.rbracket"));
}
if (matchToken(PyTokenTypes.COLON)) {
parseSingleExpression(false);
parseSingleExpression(false, true);
}
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
@@ -426,14 +424,12 @@ public class ExpressionParsing extends Parsing {
while (atToken(PyTokenTypes.COMMA)) {
nextToken();
PsiBuilder.Marker sliceItemStart = myBuilder.mark();
if (!parseEllipsis()) {
parseTestExpression(false, false);
}
parseTestExpression(false, false, true);
if (matchToken(PyTokenTypes.COLON)) {
inSlice = true;
parseTestExpression(false, false);
parseTestExpression(false, false, true);
if (matchToken(PyTokenTypes.COLON)) {
parseTestExpression(false, false);
parseTestExpression(false, false, true);
}
}
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
@@ -480,7 +476,7 @@ public class ExpressionParsing extends Parsing {
if (myBuilder.getTokenType() == PyTokenTypes.MULT || myBuilder.getTokenType() == PyTokenTypes.EXP) {
final PsiBuilder.Marker starArgMarker = myBuilder.mark();
myBuilder.advanceLexer();
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
}
starArgMarker.done(PyElementTypes.STAR_ARGUMENT_EXPRESSION);
@@ -491,7 +487,7 @@ public class ExpressionParsing extends Parsing {
myBuilder.advanceLexer();
if (myBuilder.getTokenType() == PyTokenTypes.EQ) {
myBuilder.advanceLexer();
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
}
keywordArgMarker.done(PyElementTypes.KEYWORD_ARGUMENT_EXPRESSION);
@@ -499,7 +495,7 @@ public class ExpressionParsing extends Parsing {
}
keywordArgMarker.rollbackTo();
}
if (!parseSingleExpression(false)) {
if (!parseSingleExpression(false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
break;
}
@@ -514,11 +510,11 @@ public class ExpressionParsing extends Parsing {
}
public boolean parseExpressionOptional() {
return parseTupleExpression(false, false, false);
return parseTupleExpression(false, false, false, false);
}
public boolean parseExpressionOptional(boolean isTargetExpression) {
return parseTupleExpression(false, isTargetExpression, false);
return parseTupleExpression(false, isTargetExpression, false, false);
}
public void parseExpression() {
@@ -528,7 +524,7 @@ public class ExpressionParsing extends Parsing {
}
public void parseExpression(boolean stopOnIn, boolean isTargetExpression) {
if (!parseTupleExpression(stopOnIn, isTargetExpression, false)) {
if (!parseTupleExpression(stopOnIn, isTargetExpression, false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
}
}
@@ -539,7 +535,7 @@ public class ExpressionParsing extends Parsing {
myBuilder.advanceLexer();
if (myBuilder.getTokenType() == PyTokenTypes.FROM_KEYWORD) {
myBuilder.advanceLexer();
final boolean parsed = parseTupleExpression(false, isTargetExpression, false);
final boolean parsed = parseTupleExpression(false, isTargetExpression, false, false);
if (!parsed) {
myBuilder.error(message("PARSE.expected.expression"));
}
@@ -547,19 +543,19 @@ public class ExpressionParsing extends Parsing {
return parsed;
}
else {
parseTupleExpression(false, isTargetExpression, false);
parseTupleExpression(false, isTargetExpression, false, false);
yieldExpr.done(PyElementTypes.YIELD_EXPRESSION);
return true;
}
}
else {
return parseTupleExpression(false, isTargetExpression, false);
return parseTupleExpression(false, isTargetExpression, false, false);
}
}
protected boolean parseTupleExpression(boolean stopOnIn, boolean isTargetExpression, final boolean oldTest) {
protected boolean parseTupleExpression(boolean stopOnIn, boolean isTargetExpression, final boolean oldTest, boolean allowEllipsis) {
PsiBuilder.Marker expr = myBuilder.mark();
boolean exprParseResult = oldTest ? parseOldTestExpression() : parseTestExpression(stopOnIn, isTargetExpression);
boolean exprParseResult = oldTest ? parseOldTestExpression() : parseTestExpression(stopOnIn, isTargetExpression, allowEllipsis);
if (!exprParseResult) {
expr.drop();
return false;
@@ -568,7 +564,7 @@ public class ExpressionParsing extends Parsing {
while (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
PsiBuilder.Marker expr2 = myBuilder.mark();
exprParseResult = oldTest ? parseOldTestExpression() : parseTestExpression(stopOnIn, isTargetExpression);
exprParseResult = oldTest ? parseOldTestExpression() : parseTestExpression(stopOnIn, isTargetExpression, allowEllipsis);
if (!exprParseResult) {
expr2.rollbackTo();
break;
@@ -583,8 +579,8 @@ public class ExpressionParsing extends Parsing {
return true;
}
public boolean parseSingleExpression(boolean isTargetExpression) {
return parseTestExpression(false, isTargetExpression);
public boolean parseSingleExpression(boolean isTargetExpression, boolean allowEllipsis) {
return parseTestExpression(false, isTargetExpression, allowEllipsis);
}
public boolean parseOldExpression() {
@@ -594,10 +590,13 @@ public class ExpressionParsing extends Parsing {
return parseORTestExpression(false, false);
}
private boolean parseTestExpression(boolean stopOnIn, boolean isTargetExpression) {
private boolean parseTestExpression(boolean stopOnIn, boolean isTargetExpression, boolean allowEllipsis) {
if (myBuilder.getTokenType() == PyTokenTypes.LAMBDA_KEYWORD) {
return parseLambdaExpression(false);
}
if (allowEllipsis && parseEllipsis()) {
return true;
}
PsiBuilder.Marker condExpr = myBuilder.mark();
if (!parseORTestExpression( stopOnIn, isTargetExpression)) {
condExpr.drop();
@@ -614,7 +613,7 @@ public class ExpressionParsing extends Parsing {
}
else {
myBuilder.advanceLexer();
if (!parseTestExpression(stopOnIn, isTargetExpression)) {
if (!parseTestExpression(stopOnIn, isTargetExpression, allowEllipsis)) {
myBuilder.error(message("PARSE.expected.expression"));
}
}
@@ -638,7 +637,7 @@ public class ExpressionParsing extends Parsing {
PsiBuilder.Marker expr = myBuilder.mark();
myBuilder.advanceLexer();
getFunctionParser().parseParameterListContents(PyTokenTypes.COLON, false, true);
boolean parseExpressionResult = oldTest ? parseOldTestExpression() : parseSingleExpression(false);
boolean parseExpressionResult = oldTest ? parseOldTestExpression() : parseSingleExpression(false, false);
if (!parseExpressionResult) {
myBuilder.error(message("PARSE.expected.expression"));
}
python/src/com/jetbrains/python/parsing/FunctionParsing.java
+4 -4
View File
@@ -65,7 +65,7 @@ public class FunctionParsing extends Parsing {
PsiBuilder.Marker maybeReturnAnnotation = myBuilder.mark();
nextToken();
if (matchToken(PyTokenTypes.GT)) {
if (!myContext.getExpressionParser().parseSingleExpression(false)) {
if (!myContext.getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
}
maybeReturnAnnotation.done(PyElementTypes.ANNOTATION);
@@ -195,13 +195,13 @@ public class FunctionParsing extends Parsing {
if (!isLambda && myContext.getLanguageLevel().isPy3K() && atToken(PyTokenTypes.COLON)) {
PsiBuilder.Marker annotationMarker = myBuilder.mark();
nextToken();
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
}
annotationMarker.done(PyElementTypes.ANNOTATION);
}
if (!isStarParameter && matchToken(PyTokenTypes.EQ)) {
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
PsiBuilder.Marker invalidElements = myBuilder.mark();
while(!atAnyOfTokens(endToken, PyTokenTypes.LINE_BREAK, PyTokenTypes.COMMA, null)) {
nextToken();
@@ -251,7 +251,7 @@ public class FunctionParsing extends Parsing {
}
if (myBuilder.getTokenType() == PyTokenTypes.EQ) {
myBuilder.advanceLexer();
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
}
tuple.done(PyElementTypes.TUPLE_PARAMETER);
}
python/src/com/jetbrains/python/parsing/StatementParsing.java
+17 -17
View File
@@ -273,18 +273,18 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
if (builder.getTokenType() == PyTokenTypes.GTGT) {
final PsiBuilder.Marker target = builder.mark();
builder.advanceLexer();
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
target.done(PyElementTypes.PRINT_TARGET);
}
else {
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
}
while (builder.getTokenType() == PyTokenTypes.COMMA) {
builder.advanceLexer();
if (getEndOfStatementsTokens().contains(builder.getTokenType())) {
break;
}
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
}
checkEndOfStatement(scope);
statement.done(PyElementTypes.PRINT_STATEMENT);
@@ -312,13 +312,13 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
assertCurrentToken(PyTokenTypes.DEL_KEYWORD);
final PsiBuilder.Marker delStatement = myBuilder.mark();
myBuilder.advanceLexer();
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error("Expression expected");
}
while (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
if (!getEndOfStatementsTokens().contains(myBuilder.getTokenType())) {
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error("Expression expected");
}
}
@@ -333,18 +333,18 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
final PsiBuilder.Marker raiseStatement = myBuilder.mark();
myBuilder.advanceLexer();
if (!getEndOfStatementsTokens().contains(myBuilder.getTokenType())) {
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
if (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
if (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
}
}
else if (myBuilder.getTokenType() == PyTokenTypes.FROM_KEYWORD) {
myBuilder.advanceLexer();
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error("Expression expected");
}
}
@@ -357,10 +357,10 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
assertCurrentToken(PyTokenTypes.ASSERT_KEYWORD);
final PsiBuilder.Marker assertStatement = myBuilder.mark();
myBuilder.advanceLexer();
if (getExpressionParser().parseSingleExpression(false)) {
if (getExpressionParser().parseSingleExpression(false, false)) {
if (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myContext.getBuilder().error(EXPRESSION_EXPECTED);
}
}
@@ -557,10 +557,10 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
getExpressionParser().parseExpression(true, false);
if (myBuilder.getTokenType() == PyTokenTypes.IN_KEYWORD) {
myBuilder.advanceLexer();
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
if (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
getExpressionParser().parseSingleExpression(false);
getExpressionParser().parseSingleExpression(false, false);
}
}
checkEndOfStatement(inSuite);
@@ -641,7 +641,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
final PsiBuilder.Marker statement = myBuilder.mark();
final PsiBuilder.Marker whilePart = myBuilder.mark();
myBuilder.advanceLexer();
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error(EXPRESSION_EXPECTED);
}
parseColonAndSuite(scope);
@@ -672,13 +672,13 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
final PsiBuilder.Marker exceptBlock = myBuilder.mark();
myBuilder.advanceLexer();
if (myBuilder.getTokenType() != PyTokenTypes.COLON) {
if (!getExpressionParser().parseSingleExpression(false)) {
if (!getExpressionParser().parseSingleExpression(false, false)) {
myBuilder.error(EXPRESSION_EXPECTED);
}
myExpectAsKeyword = true;
if (myBuilder.getTokenType() == PyTokenTypes.COMMA || myBuilder.getTokenType() == PyTokenTypes.AS_KEYWORD) {
myBuilder.advanceLexer();
if (!getExpressionParser().parseSingleExpression(true)) {
if (!getExpressionParser().parseSingleExpression(true, false)) {
myBuilder.error(EXPRESSION_EXPECTED);
}
}
@@ -733,7 +733,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper {
myExpectAsKeyword = true;
if (myBuilder.getTokenType() == PyTokenTypes.AS_KEYWORD) {
myBuilder.advanceLexer();
getExpressionParser().parseSingleExpression(true); // 'as' is followed by a target
getExpressionParser().parseSingleExpression(true, false); // 'as' is followed by a target
}
withItem.done(PyElementTypes.WITH_ITEM);
if (!matchToken(PyTokenTypes.COMMA)) {
python/testData/psi/EllipsisInSliceList.py
+1
View File
@@ -0,0 +1 @@
y = x[..., 0]
python/testData/psi/EllipsisInSliceList.txt
+21
View File
@@ -0,0 +1,21 @@
PyFile:EllipsisInSliceList.py
PyAssignmentStatement
PyTargetExpression: y
PsiElement(Py:IDENTIFIER)('y')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PySubscriptionExpression
PyReferenceExpression: x
PsiElement(Py:IDENTIFIER)('x')
PsiElement(Py:LBRACKET)('[')
PyTupleExpression
PyNoneLiteralExpression
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('0')
PsiElement(Py:RBRACKET)(']')
python/testData/psi/EllipsisInSliceListTail.py
+1
View File
@@ -0,0 +1 @@
y = x[0, ..., 2]
python/testData/psi/EllipsisInSliceListTail.txt
+25
View File
@@ -0,0 +1,25 @@
PyFile:EllipsisInSliceListTail.py
PyAssignmentStatement
PyTargetExpression: y
PsiElement(Py:IDENTIFIER)('y')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
PsiWhiteSpace(' ')
PySubscriptionExpression
PyReferenceExpression: x
PsiElement(Py:IDENTIFIER)('x')
PsiElement(Py:LBRACKET)('[')
PyTupleExpression
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('0')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyNoneLiteralExpression
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiElement(Py:RBRACKET)(']')
python/testSrc/com/jetbrains/python/PythonParsingTest.java
+10
View File
@@ -360,6 +360,16 @@ public class PythonParsingTest extends ParsingTestCase {
doTest();
}
// PY-8606
public void testEllipsisInSliceList() {
doTest();
}
// PY-8606
public void testEllipsisInSliceListTail() {
doTest();
}
public void doTest(LanguageLevel languageLevel) {
LanguageLevel prev = myLanguageLevel;
myLanguageLevel = languageLevel;