PY-81646 False positive: Unresolved references after any slice in Python for the rest of the line

Fixed AST for PySubscriptionExpression containing a slice.
Before PySubscriptionExpression.getIndexExpression() returned only the first slice expression (if any).

Also fixed PySliceItem's subtree, so now its children are correctly mapped to lowerBound, upperBound and stride properties.

GitOrigin-RevId: 840af92957d0431679cf7a477866dea76320ca80
This commit is contained in:
Petr
2025-06-05 18:20:37 +02:00
committed by intellij-monorepo-bot
parent 76126e7179
commit ff19de5654
7 changed files with 141 additions and 209 deletions

View File

@@ -2154,6 +2154,15 @@ public abstract class PyCommonResolveTest extends PyCommonResolveTestCase {
""", PyTargetExpression.class, "Alias");
}
// PY-81646
public void testResolveFromSliceExpression() {
assertResolvesTo("""
def foo(arr, idx):
_ = arr[:, idx]
# <ref>
""", PyNamedParameter.class, "idx");
}
private void assertResolvedElement(@NotNull LanguageLevel languageLevel, @NotNull String text, @NotNull Consumer<PsiElement> assertion) {
runWithLanguageLevel(languageLevel, () -> {
myFixture.configureByText(PythonFileType.INSTANCE, text);

View File

@@ -6,7 +6,6 @@ import com.intellij.lang.WhitespacesAndCommentsBinder;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.NlsContexts.ParsingError;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyTokenTypes;
import org.jetbrains.annotations.NotNull;
@@ -306,7 +305,7 @@ public class ExpressionParsing extends Parsing {
result = parseORTestExpression(false, false);
}
else {
result = parseTupleExpression(false, false, true);
result = parseTupleExpression(false, false, true, false);
}
if (!result) {
myBuilder.error(message("PARSE.expected.expression"));
@@ -515,7 +514,8 @@ public class ExpressionParsing extends Parsing {
}
else if (tokenType == PyTokenTypes.LBRACKET) {
myBuilder.advanceLexer();
parseSliceOrSubscriptionExpression(expr, false);
parseSubscriptionIndexArgumentList();
expr.done(PyElementTypes.SUBSCRIPTION_EXPRESSION);
if (isTargetExpression && !recastQualifier) {
recastFirstIdentifier = true; // subscription is always a reference
recastQualifier = true; // recast non-first qualifiers too
@@ -537,47 +537,11 @@ public class ExpressionParsing extends Parsing {
return true;
}
private void parseSliceOrSubscriptionExpression(@NotNull SyntaxTreeBuilder.Marker expr, boolean isSlice) {
SyntaxTreeBuilder.Marker sliceOrTupleStart = myBuilder.mark();
SyntaxTreeBuilder.Marker sliceItemStart = myBuilder.mark();
if (atToken(PyTokenTypes.COLON)) {
sliceOrTupleStart.drop();
SyntaxTreeBuilder.Marker sliceMarker = myBuilder.mark();
sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION);
parseSliceEnd(expr, sliceItemStart);
}
else {
var hadExpression = isSlice ? parseSingleExpression(false) : parseNamedTestExpression(false, false);
if (atToken(PyTokenTypes.COLON)) {
if (!isSlice) {
sliceOrTupleStart.rollbackTo();
parseSliceOrSubscriptionExpression(expr, true);
return;
}
sliceOrTupleStart.drop();
parseSliceEnd(expr, sliceItemStart);
}
else if (atToken(PyTokenTypes.COMMA)) {
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
if (!parseSliceListTail(expr, sliceOrTupleStart)) {
sliceOrTupleStart.rollbackTo();
if (!parseTupleExpression(false, false, false)) {
myBuilder.error(message("tuple.expression.expected"));
}
checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket"));
expr.done(PyElementTypes.SUBSCRIPTION_EXPRESSION);
}
}
else {
if (!hadExpression) {
myBuilder.error(message("PARSE.expected.expression"));
}
sliceOrTupleStart.drop();
sliceItemStart.drop();
checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket"));
expr.done(PyElementTypes.SUBSCRIPTION_EXPRESSION);
}
public void parseSubscriptionIndexArgumentList() {
if (!parseTupleExpression(false, false, false, true)) {
myBuilder.error(message("PARSE.expected.expression"));
}
checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket"));
}
private boolean parseEllipsis() {
@@ -593,73 +557,6 @@ public class ExpressionParsing extends Parsing {
return false;
}
private static final TokenSet BRACKET_OR_COMMA = TokenSet.create(PyTokenTypes.RBRACKET, PyTokenTypes.COMMA);
private static final TokenSet BRACKET_COLON_COMMA = TokenSet.create(PyTokenTypes.RBRACKET, PyTokenTypes.COLON, PyTokenTypes.COMMA);
public void parseSliceEnd(SyntaxTreeBuilder.Marker exprStart, SyntaxTreeBuilder.Marker sliceItemStart) {
myBuilder.advanceLexer();
if (atToken(PyTokenTypes.RBRACKET)) {
SyntaxTreeBuilder.Marker sliceMarker = myBuilder.mark();
sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION);
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
nextToken();
exprStart.done(PyElementTypes.SUBSCRIPTION_EXPRESSION);
return;
}
else {
if (atToken(PyTokenTypes.COLON)) {
SyntaxTreeBuilder.Marker sliceMarker = myBuilder.mark();
sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION);
}
else {
parseSingleExpression(false);
}
if (!BRACKET_COLON_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error(message("PARSE.expected.colon.or.rbracket"));
}
if (matchToken(PyTokenTypes.COLON)) {
parseSingleExpression(false);
}
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
if (!BRACKET_OR_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error(message("rbracket.or.comma.expected"));
}
}
parseSliceListTail(exprStart, null);
}
private boolean parseSliceListTail(SyntaxTreeBuilder.Marker exprStart, @Nullable SyntaxTreeBuilder.Marker sliceOrTupleStart) {
boolean inSlice = sliceOrTupleStart == null;
while (atToken(PyTokenTypes.COMMA)) {
nextToken();
SyntaxTreeBuilder.Marker sliceItemStart = myBuilder.mark();
parseTestExpression(false, false);
if (matchToken(PyTokenTypes.COLON)) {
inSlice = true;
parseTestExpression(false, false);
if (matchToken(PyTokenTypes.COLON)) {
parseTestExpression(false, false);
}
}
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
if (!BRACKET_OR_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error(message("rbracket.or.comma.expected"));
break;
}
}
checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket"));
if (inSlice) {
if (sliceOrTupleStart != null) {
sliceOrTupleStart.drop();
}
exprStart.done(PyElementTypes.SUBSCRIPTION_EXPRESSION);
}
return inSlice;
}
public void parseArgumentList() {
LOG.assertTrue(myBuilder.getTokenType() == PyTokenTypes.LPAR);
final SyntaxTreeBuilder.Marker arglist = myBuilder.mark();
@@ -721,11 +618,11 @@ public class ExpressionParsing extends Parsing {
}
public boolean parseExpressionOptional() {
return parseTupleExpression(false, false, false);
return parseTupleExpression(false, false, false, false);
}
public boolean parseExpressionOptional(boolean isTargetExpression) {
return parseTupleExpression(false, isTargetExpression, false);
return parseTupleExpression(false, isTargetExpression, false, false);
}
public void parseExpression() {
@@ -735,7 +632,7 @@ public class ExpressionParsing extends Parsing {
}
public void parseExpression(boolean stopOnIn, boolean isTargetExpression) {
if (!parseTupleExpression(stopOnIn, isTargetExpression, false)) {
if (!parseTupleExpression(stopOnIn, isTargetExpression, false, false)) {
myBuilder.error(message("PARSE.expected.expression"));
}
}
@@ -746,7 +643,7 @@ public class ExpressionParsing extends Parsing {
myBuilder.advanceLexer();
if (myBuilder.getTokenType() == PyTokenTypes.FROM_KEYWORD) {
myBuilder.advanceLexer();
final boolean parsed = parseTupleExpression(false, isTargetExpression, false);
final boolean parsed = parseTupleExpression(false, isTargetExpression, false, false);
if (!parsed) {
myBuilder.error(message("PARSE.expected.expression"));
}
@@ -754,19 +651,21 @@ public class ExpressionParsing extends Parsing {
return parsed;
}
else {
parseTupleExpression(false, isTargetExpression, false);
parseTupleExpression(false, isTargetExpression, false, false);
yieldExpr.done(PyElementTypes.YIELD_EXPRESSION);
return true;
}
}
else {
return parseTupleExpression(false, isTargetExpression, false);
return parseTupleExpression(false, isTargetExpression, false, false);
}
}
protected boolean parseTupleExpression(boolean stopOnIn, boolean isTargetExpression, final boolean oldTest) {
protected boolean parseTupleExpression(boolean stopOnIn, boolean isTargetExpression, final boolean oldTest, boolean isSubscription) {
SyntaxTreeBuilder.Marker expr = myBuilder.mark();
boolean exprParseResult = oldTest ? parseOldTestExpression() : parseNamedTestExpression(stopOnIn, isTargetExpression);
boolean exprParseResult = isSubscription ? parseSubscriptionIndexArgument() :
oldTest ? parseOldTestExpression() :
parseNamedTestExpression(stopOnIn, isTargetExpression);
if (!exprParseResult) {
expr.drop();
return false;
@@ -775,7 +674,9 @@ public class ExpressionParsing extends Parsing {
while (myBuilder.getTokenType() == PyTokenTypes.COMMA) {
myBuilder.advanceLexer();
SyntaxTreeBuilder.Marker expr2 = myBuilder.mark();
exprParseResult = oldTest ? parseOldTestExpression() : parseNamedTestExpression(stopOnIn, isTargetExpression);
exprParseResult = isSubscription ? parseSubscriptionIndexArgument() :
oldTest ? parseOldTestExpression() :
parseNamedTestExpression(stopOnIn, isTargetExpression);
if (!exprParseResult) {
expr2.rollbackTo();
break;
@@ -790,6 +691,27 @@ public class ExpressionParsing extends Parsing {
return true;
}
private boolean parseSubscriptionIndexArgument() {
SyntaxTreeBuilder.Marker sliceItem = myBuilder.mark();
if (!parseSingleExpression(false)) {
myBuilder.mark().done(PyElementTypes.EMPTY_EXPRESSION);
}
if (!matchToken(PyTokenTypes.COLON)) {
sliceItem.rollbackTo();
return parseNamedTestExpression(false, false);
}
boolean exprParseResult = parseSingleExpression(false);
if (myBuilder.getTokenType() == PyTokenTypes.COLON) {
if (!exprParseResult) {
myBuilder.mark().done(PyElementTypes.EMPTY_EXPRESSION);
}
myBuilder.advanceLexer();
parseSingleExpression(false);
}
sliceItem.done(PyElementTypes.SLICE_ITEM);
return true;
}
public boolean parseSingleExpression(boolean isTargetExpression) {
return parseTestExpression(false, isTargetExpression);
}

View File

@@ -14,7 +14,7 @@ def c(d):
e = None # type: Union[str<EOLError descr="']' expected"></EOLError>
def f(g: Union[int<error descr="']' expected">)</error> -> Union[str:<EOLError descr="':' or ']' expected"></EOLError>
def f(g: Union[int<error descr="']' expected">)</error> -> Union[str:<EOLError descr="']' expected"></EOLError>
pass<EOLError descr="End of statement expected"></EOLError>

View File

@@ -86,36 +86,35 @@ PyFile:AssignmentExpressionsInIndexes.py
PsiWhiteSpace(' ')
PsiComment(Py:END_OF_LINE_COMMENT)('# valid')
PsiWhiteSpace('\n')
PyAssignmentExpression
PyTypeDeclarationStatement
PySubscriptionExpression
PyReferenceExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiElement(Py:LBRACKET)('[')
PyReferenceExpression: d
PsiElement(Py:IDENTIFIER)('d')
PsiErrorElement:']' expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('0')
PsiElement(Py:COLON)(':')
PsiErrorElement:Statement expected, found Py:IDENTIFIER
<empty list>
PsiWhiteSpace(' ')
PyExpressionStatement
PyParenthesizedExpression
PsiElement(Py:LPAR)('(')
PyAssignmentExpression
PyTargetExpression: e
PsiElement(Py:IDENTIFIER)('e')
PyTargetExpression: d
PsiElement(Py:IDENTIFIER)('d')
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiElement(Py:RPAR)(')')
PsiElement(Py:INTEGER_LITERAL)('0')
PsiErrorElement:']' expected
<empty list>
PyAnnotation
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyParenthesizedExpression
PsiElement(Py:LPAR)('(')
PyAssignmentExpression
PyTargetExpression: e
PsiElement(Py:IDENTIFIER)('e')
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiElement(Py:RPAR)(')')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACKET)(']')
@@ -124,33 +123,36 @@ PyFile:AssignmentExpressionsInIndexes.py
PsiWhiteSpace(' ')
PsiComment(Py:END_OF_LINE_COMMENT)('# invalid')
PsiWhiteSpace('\n')
PyAssignmentExpression
PyTypeDeclarationStatement
PySubscriptionExpression
PyReferenceExpression: s
PsiElement(Py:IDENTIFIER)('s')
PsiElement(Py:LBRACKET)('[')
PyReferenceExpression: d
PsiElement(Py:IDENTIFIER)('d')
PyAssignmentExpression
PyTargetExpression: d
PsiElement(Py:IDENTIFIER)('d')
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('0')
PsiErrorElement:']' expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('0')
PsiElement(Py:COLON)(':')
PsiErrorElement:Statement expected, found Py:IDENTIFIER
PyAnnotation
PsiElement(Py:COLON)(':')
PsiWhiteSpace(' ')
PyReferenceExpression: e
PsiElement(Py:IDENTIFIER)('e')
PsiErrorElement:End of statement expected
<empty list>
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiErrorElement:Statement expected, found Py:COLONEQ
<empty list>
PsiWhiteSpace(' ')
PyExpressionStatement
PyAssignmentExpression
PyTargetExpression: e
PsiElement(Py:IDENTIFIER)('e')
PsiWhiteSpace(' ')
PsiElement(Py:COLONEQ)(':=')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiErrorElement:End of statement expected
<empty list>
PsiElement(Py:RBRACKET)(']')

View File

@@ -52,8 +52,6 @@ PyFile:ExtendedSlices.py
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('0')
PsiElement(Py:COLON)(':')
PyEmptyExpression
<empty list>
PsiElement(Py:RBRACKET)(']')
PsiWhiteSpace('\n')
PyAssignmentStatement
@@ -61,26 +59,27 @@ PyFile:ExtendedSlices.py
PyReferenceExpression: d
PsiElement(Py:IDENTIFIER)('d')
PsiElement(Py:LBRACKET)('[')
PySliceItem
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('3')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('3')
PyTupleExpression
PySliceItem
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('3')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('1')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('2')
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('3')
PsiElement(Py:RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')
@@ -93,34 +92,34 @@ PyFile:ExtendedSlices.py
PyReferenceExpression: d
PsiElement(Py:IDENTIFIER)('d')
PsiElement(Py:LBRACKET)('[')
PySliceItem
PyEmptyExpression
<empty list>
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PyTupleExpression
PySliceItem
PyEmptyExpression
<empty list>
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('42')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyEllipsisLiteralExpression
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:DOT)('.')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PsiElement(Py:COLON)(':')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PyEmptyExpression
<empty list>
PsiElement(Py:COLON)(':')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('24')
PsiElement(Py:COLON)(':')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('24')
PsiElement(Py:COLON)(':')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('24')
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PySliceItem
PsiElement(Py:COMMA)(',')
PsiWhiteSpace(' ')
PyNumericLiteralExpression
PsiElement(Py:INTEGER_LITERAL)('100')
PsiElement(Py:RBRACKET)(']')

View File

@@ -18,8 +18,6 @@ PyFile:RangeAsLHS.py
PyEmptyExpression
<empty list>
PsiElement(Py:COLON)(':')
PyEmptyExpression
<empty list>
PsiElement(Py:RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(Py:EQ)('=')

View File

@@ -4,10 +4,12 @@ PyFile:SliceList.py
PyReferenceExpression: a
PsiElement(Py:IDENTIFIER)('a')
PsiElement(Py:LBRACKET)('[')
PySliceItem
PyTupleExpression
PyReferenceExpression: b1
PsiElement(Py:IDENTIFIER)('b1')
PsiElement(Py:COMMA)(',')
PySliceItem
PsiElement(Py:COLON)(':')
PsiElement(Py:COMMA)(',')
PySliceItem
PyEmptyExpression
<empty list>
PsiElement(Py:COLON)(':')
PsiElement(Py:RBRACKET)(']')