[PY-49040] add support for TypeGuards

Merge-request: IJ-MR-109904
Merged-by: Vladimir Koshelev <Vladimir.Koshelev@jetbrains.com>

GitOrigin-RevId: 8441ce35b2fc97fb0cdaf747feff2cf9ba3347ea
This commit is contained in:
Vladimir Koshelev
2023-08-02 22:27:54 +00:00
committed by intellij-monorepo-bot
parent 15582d4a59
commit 7aeb74fe0a
13 changed files with 416 additions and 18 deletions
@@ -48,6 +48,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
private static final Set<String> EXCEPTION_SUPPRESSORS = ImmutableSet.of("suppress", "assertRaises", "assertRaisesRegex");
private final ControlFlowBuilder myBuilder = new ControlFlowBuilder();
private final Map<PyExpression, PyFunction> expressionToGuards = new HashMap<>();
public ControlFlow buildControlFlow(@NotNull final ScopeOwner owner) {
return myBuilder.build(this, owner);
@@ -58,6 +59,31 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
return this.myBuilder;
}
private void startConditionalNodeAndCheckGuards(@NotNull PsiElement element, @Nullable PyExpression condition, boolean result) {
myBuilder.startConditionalNode(element, condition, result);
addTypeGuardAssertions(condition, result);
}
private void addTypeGuardAssertions(@Nullable PyExpression condition, boolean result) {
final PyExpression actualExpression;
final boolean negation;
if (condition instanceof PyPrefixExpression prefixExpression && prefixExpression.getOperator() == PyTokenTypes.NOT_KEYWORD) {
actualExpression = prefixExpression.getOperand();
negation = true;
}
else {
actualExpression = condition;
negation = false;
}
var function = expressionToGuards.get(actualExpression);
if ((negation && !result || !negation && result) && function != null && actualExpression instanceof PyCallExpression callExpression) {
final var evaluator = new PyTypeAssertionEvaluator();
evaluator.handleTypeGuardCall(callExpression, function);
InstructionBuilder.addAssertInstructions(myBuilder, evaluator);
}
}
@Override
public void visitPyFunction(final @NotNull PyFunction node) {
// Create node and stop here
@@ -133,17 +159,24 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
@Override
public void visitPyCallExpression(final @NotNull PyCallExpression node) {
final PyExpression callee = node.getCallee();
final var callNodeType = getCalleeNodeType(callee);
// Flow abrupted
if (callee != null && isCallOfNoReturnFunction(callee)) {
if (callNodeType instanceof NoReturnCallKind) {
callee.accept(this);
for (PyExpression expression : node.getArguments()) {
expression.accept(this);
}
abruptFlow(node);
}
else if (callNodeType instanceof TypeGuardCallKind typeGuardCallKind && node.getArguments().length > 0) {
expressionToGuards.put(node, typeGuardCallKind.pyFunction);
super.visitPyCallExpression(node);
}
else {
super.visitPyCallExpression(node);
}
if (node.isCalleeText(PyNames.ASSERT_IS_INSTANCE)) {
final PyTypeAssertionEvaluator assertionEvaluator = new PyTypeAssertionEvaluator();
node.accept(assertionEvaluator);
@@ -372,7 +405,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
}
myBuilder.prevInstruction = null;
myBuilder.startConditionalNode(part, lastCondition, false);
startConditionalNodeAndCheckGuards(part, lastCondition, false);
}
final Triple<PyExpression, List<Pair<PsiElement, Instruction>>, Boolean> currentPartResults = visitPyConditionalPart(part, node);
@@ -398,7 +431,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
final PyStatementList statements = elseBranch.getStatementList();
myBuilder.startConditionalNode(statements, lastCondition, false);
startConditionalNodeAndCheckGuards(statements, lastCondition, false);
InstructionBuilder.addAssertInstructions(myBuilder, negativeAssertionEvaluator);
statements.accept(this);
@@ -463,7 +496,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
@NotNull PyStatement node) {
final PyStatementList statements = part.getStatementList();
myBuilder.startConditionalNode(statements, part.getCondition(), true);
startConditionalNodeAndCheckGuards(statements, part.getCondition(), true);
InstructionBuilder.addAssertInstructions(myBuilder, assertionEvaluator);
statements.accept(this);
@@ -534,10 +567,12 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
final var outside = new ConditionalInstructionImpl(myBuilder, null, subExpression, !conditionResultToContinue);
myBuilder.addNode(outside);
myBuilder.addPendingEdge(node, outside);
addTypeGuardAssertions(subExpression, !conditionResultToContinue);
myBuilder.addPendingEdge(node, myBuilder.prevInstruction);
myBuilder.prevInstruction = branchingPoint;
final var toTheNext = new ConditionalInstructionImpl(myBuilder, null, subExpression, conditionResultToContinue);
addTypeGuardAssertions(subExpression, conditionResultToContinue);
myBuilder.addNode(toTheNext);
}
@@ -568,6 +603,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
final var elsePartInstruction = new ConditionalInstructionImpl(myBuilder, elsePart, mainPartResults.getFirst(), false);
myBuilder.prevInstruction = null;
myBuilder.addNode(elsePartInstruction);
addTypeGuardAssertions(mainPartResults.getFirst(), false);
if (!isStaticallyTrue) {
for (Pair<PsiElement, Instruction> pair : branchingPoints) {
@@ -874,7 +910,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
final PyExpression iteratedList = c.getIteratedList();
final PyExpression iteratorVariable = c.getIteratorVariable();
if (prevCondition != null) {
myBuilder.startConditionalNode(iteratedList, prevCondition, true);
startConditionalNodeAndCheckGuards(iteratedList, prevCondition, true);
prevCondition = null;
}
else {
@@ -901,7 +937,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
continue;
}
if (prevCondition != null) {
myBuilder.startConditionalNode(condition, prevCondition, true);
startConditionalNodeAndCheckGuards(condition, prevCondition, true);
}
else {
myBuilder.startNode(condition);
@@ -925,7 +961,7 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
final PyExpression result = node.getResultExpression();
if (result != null) {
if (prevCondition != null) {
myBuilder.startConditionalNode(result, prevCondition, true);
startConditionalNodeAndCheckGuards(result, prevCondition, true);
}
else {
myBuilder.startNode(result);
@@ -992,12 +1028,13 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
if (target != null) target.accept(this);
}
private static boolean isCallOfNoReturnFunction(@NotNull PyExpression callee) {
@Nullable
private static CallTypeKind getCalleeNodeType(@Nullable PyExpression callee) {
if (callee instanceof PyReferenceExpression expression) {
QualifiedName qName = expression.asQualifiedName();
if (qName == null) {
return false;
return null;
}
ScopeOwner scopeOwner = ScopeUtil.getScopeOwner(expression);
@@ -1006,18 +1043,23 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
TypeEvalContext context = TypeEvalContext.codeInsightFallback(callee.getProject());
while (scopeOwner != null) {
boolean resolvesToNoReturnOrNever = StreamEx
final var result = StreamEx
.of(PyResolveUtil.resolveQualifiedNameInScope(qName, scopeOwner, context))
.select(PyFunction.class)
.anyMatch(function -> PyTypingTypeProvider.isNoReturn(function, context));
if (resolvesToNoReturnOrNever) {
return true;
}
.map(function -> {
if (PyTypingTypeProvider.isNoReturn(function, context)) {
return NoReturnCallKind.INSTANCE;
}
if (PyTypingTypeProvider.isTypeGuard(function, context)) {
return new TypeGuardCallKind(function);
}
return null;
}).findFirst( it -> it != null);
if (result.isPresent()) return result.get();
scopeOwner = ScopeUtil.getScopeOwner(scopeOwner);
}
}
return false;
return null;
}
private void abruptFlow(final PsiElement node) {
@@ -1041,5 +1083,14 @@ public class PyControlFlowBuilder extends PyRecursiveElementVisitor {
return !PsiTreeUtil.instanceOf(instruction.getElement(),
PyStatementList.class);
}
private interface CallTypeKind { }
private static class NoReturnCallKind implements CallTypeKind {
private NoReturnCallKind() {};
public static final NoReturnCallKind INSTANCE = new NoReturnCallKind();
}
private record TypeGuardCallKind(@NotNull PyFunction pyFunction) implements CallTypeKind {}
}
@@ -6,8 +6,10 @@ import com.intellij.psi.PsiElement;
import com.intellij.util.containers.Stack;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.codeInsight.functionTypeComments.psi.PyFunctionTypeAnnotation;
import com.jetbrains.python.codeInsight.typing.PyTypingTypeProvider;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.impl.PyBuiltinCache;
import com.jetbrains.python.psi.impl.PyEvaluator;
import com.jetbrains.python.psi.impl.PyPsiUtils;
import com.jetbrains.python.psi.types.*;
@@ -46,6 +48,29 @@ public class PyTypeAssertionEvaluator extends PyRecursiveElementVisitor {
}
}
public void handleTypeGuardCall(@NotNull PyCallExpression call, @NotNull PyFunction function) {
if (call.getArguments().length == 0) return;
final var firstArgument = call.getArguments()[0];
final var annotation = function.getAnnotationValue();
if (annotation == null) return;
if (firstArgument instanceof PyReferenceExpression referenceExpression) {
pushAssertion(referenceExpression, myPositive, false, (context) -> {
var returnType = PyTypingTypeProvider.getReturnTypeAnnotation(function, context);
if (returnType instanceof PyStringLiteralExpression stringLiteralExpression) {
returnType = PyUtil.createExpressionFromFragment(stringLiteralExpression.getStringValue(),
function.getContainingFile());
}
if (returnType instanceof PySubscriptionExpression subscriptionExpression) {
var indexExpression = subscriptionExpression.getIndexExpression();
if (indexExpression != null) {
return Ref.deref(PyTypingTypeProvider.getType(indexExpression, context));
}
}
return null;
}, null);
}
}
@Override
public void visitPyCallExpression(@NotNull PyCallExpression node) {
if (node.isCalleeText(PyNames.ISINSTANCE, PyNames.ASSERT_IS_INSTANCE)) {
@@ -195,6 +220,9 @@ public class PyTypeAssertionEvaluator extends PyRecursiveElementVisitor {
return type;
}
/**
* @param transformToDefinition if true the result type will be Type[T], not T itself.
*/
private void pushAssertion(@NotNull PyReferenceExpression target,
boolean positive,
boolean transformToDefinition,
@@ -62,6 +62,8 @@ public class PyTypingTypeProvider extends PyTypeProviderWithCustomContext<PyTypi
public static final String NAMEDTUPLE = "typing.NamedTuple";
public static final String TYPED_DICT = "typing.TypedDict";
public static final String TYPED_DICT_EXT = "typing_extensions.TypedDict";
public static final String TYPE_GUARD = "typing.TypeGuard";
public static final String TYPE_GUARD_EXT = "typing_extensions.TypeGuard";
public static final String GENERIC = "typing.Generic";
public static final String PROTOCOL = "typing.Protocol";
public static final String PROTOCOL_EXT = "typing_extensions.Protocol";
@@ -1125,7 +1127,13 @@ public class PyTypingTypeProvider extends PyTypeProviderWithCustomContext<PyTypi
private static <T extends PyTypeCommentOwner & PyAnnotationOwner> boolean typeHintedWithName(@NotNull T owner,
@NotNull TypeEvalContext context,
String... names) {
final PyExpression annotation = getAnnotationValue(owner, context);
var annotation = getAnnotationValue(owner, context);
if (annotation instanceof PyStringLiteralExpression stringLiteralExpression) {
final var annotationText = stringLiteralExpression.getStringValue();
annotation = toExpression(annotationText, owner);
if (annotation == null) return false;
}
if (annotation instanceof PySubscriptionExpression) {
return resolvesToQualifiedNames(((PySubscriptionExpression)annotation).getOperand(), context, names);
}
@@ -1165,6 +1173,11 @@ public class PyTypingTypeProvider extends PyTypeProviderWithCustomContext<PyTypi
typeHintedWithName(function, context, NO_RETURN, NO_RETURN_EXT, NEVER, NEVER_EXT));
}
public static boolean isTypeGuard(@NotNull PyFunction function, @NotNull TypeEvalContext context) {
return PyUtil.getParameterizedCachedValue(function, context, p ->
typeHintedWithName(function, context, TYPE_GUARD, TYPE_GUARD_EXT));
}
private static boolean resolvesToQualifiedNames(@NotNull PyExpression expression, @NotNull TypeEvalContext context, String... names) {
final var qualifiedNames = resolveToQualifiedNames(expression, context);
return ContainerUtil.exists(names, qualifiedNames::contains);
@@ -0,0 +1,13 @@
from typing import List
from typing_extensions import TypeGuard
import foo
def checkit(foo: List[int]) -> TypeGuard[List[str]]:
pass
x = foo.bar()
if checkit(x):
print(x)
else:
pass
@@ -0,0 +1,29 @@
0(1) element: null
1(2) element: PyFromImportStatement
2(3) WRITE ACCESS: List
3(4) element: PyFromImportStatement
4(5) WRITE ACCESS: TypeGuard
5(6) element: PyImportStatement
6(7) WRITE ACCESS: foo
7(8) element: PyFunction('checkit')
8(9) element: PySubscriptionExpression
9(10) READ ACCESS: List
10(11) READ ACCESS: int
11(12) element: PySubscriptionExpression
12(13) READ ACCESS: TypeGuard
13(14) element: PySubscriptionExpression
14(15) READ ACCESS: List
15(16) READ ACCESS: str
16(17) WRITE ACCESS: checkit
17(18) element: PyAssignmentStatement
18(19) READ ACCESS: foo
19(20) WRITE ACCESS: x
20(21) element: PyIfStatement
21(22) READ ACCESS: checkit
22(23,27) READ ACCESS: x
23(24) element: PyStatementList. Condition: checkit(x):true
24(25) ASSERTTYPE ACCESS: x
25(26) element: PyPrintStatement
26(28) READ ACCESS: x
27(28) element: PyStatementList. Condition: checkit(x):false
28() element: null
@@ -0,0 +1,9 @@
from typing import List
from typing_extensions import TypeGuard
import foo
def checkit(foo: List[int]) -> TypeGuard[List[str]]:
pass
x = foo.bar()
y = checkit(foo) and foo[123]
@@ -0,0 +1,33 @@
0(1) element: null
1(2) element: PyFromImportStatement
2(3) WRITE ACCESS: List
3(4) element: PyFromImportStatement
4(5) WRITE ACCESS: TypeGuard
5(6) element: PyImportStatement
6(7) WRITE ACCESS: foo
7(8) element: PyFunction('checkit')
8(9) element: PySubscriptionExpression
9(10) READ ACCESS: List
10(11) READ ACCESS: int
11(12) element: PySubscriptionExpression
12(13) READ ACCESS: TypeGuard
13(14) element: PySubscriptionExpression
14(15) READ ACCESS: List
15(16) READ ACCESS: str
16(17) WRITE ACCESS: checkit
17(18) element: PyAssignmentStatement
18(19) READ ACCESS: foo
19(20) WRITE ACCESS: x
20(21) element: PyAssignmentStatement
21(22) element: PyBinaryExpression
22(23) READ ACCESS: checkit
23(24,26) READ ACCESS: foo
24(31) element: null. Condition: checkit(foo):false
26(25) ASSERTTYPE ACCESS: foo
25(27) element: null. Condition: checkit(foo):true
27(28) element: PySubscriptionExpression
28(29,30) READ ACCESS: foo
29(31) element: null. Condition: foo[123]:false
30(31) element: null. Condition: foo[123]:true
31(32) WRITE ACCESS: y
32() element: null
@@ -0,0 +1,10 @@
from typing import List
from typing_extensions import TypeGuard
import foo
def checkit(foo: List[int]) -> TypeGuard[List[str]]:
pass
x = foo.bar()
while checkit(x):
x = foo.bar()
@@ -0,0 +1,29 @@
0(1) element: null
1(2) element: PyFromImportStatement
2(3) WRITE ACCESS: List
3(4) element: PyFromImportStatement
4(5) WRITE ACCESS: TypeGuard
5(6) element: PyImportStatement
6(7) WRITE ACCESS: foo
7(8) element: PyFunction('checkit')
8(9) element: PySubscriptionExpression
9(10) READ ACCESS: List
10(11) READ ACCESS: int
11(12) element: PySubscriptionExpression
12(13) READ ACCESS: TypeGuard
13(14) element: PySubscriptionExpression
14(15) READ ACCESS: List
15(16) READ ACCESS: str
16(17) WRITE ACCESS: checkit
17(18) element: PyAssignmentStatement
18(19) READ ACCESS: foo
19(20) WRITE ACCESS: x
20(21) element: PyWhileStatement
21(22) READ ACCESS: checkit
22(23,28) READ ACCESS: x
23(24) element: PyStatementList. Condition: checkit(x):true
24(25) ASSERTTYPE ACCESS: x
25(26) element: PyAssignmentStatement
26(27) READ ACCESS: foo
27(20) WRITE ACCESS: x
28() element: null
@@ -0,0 +1,7 @@
from typing import NoReturn
def stop() -> "NoReturn":
raise RuntimeError('no way')
stop()
<warning descr="This code is unreachable">print("Should be reported as unreachable")</warning>
@@ -1966,6 +1966,166 @@ public class Py3TypeTest extends PyTestCase {
);
}
public void testTypeGuardList() {
doTest("list[str]",
"""
from typing import List
from typing import TypeGuard
def is_str_list(val: List[object]) -> TypeGuard[List[str]]:
return all(isinstance(x, str) for x in val)
def func1(val: List[object]):
if is_str_list(val):
expr = val
""");
}
public void testTypeGuardListInStringLiteral() {
doTest("list[str]",
"""
from typing import List
from typing import TypeGuard
def is_str_list(val: List[object]) -> "TypeGuard[List[str]]":
return all(isinstance(x, str) for x in val)
def func1(val: List[object]):
if is_str_list(val):
expr = val
""");
}
public void testTypeGuardListTypeIsNotChanged() {
doTest("list[object]",
"""
from typing import List
from typing import TypeGuard
def is_str_list(val: List[object]) -> TypeGuard[List[str]]:
return all(isinstance(x, str) for x in val)
def func1(val: List[object]):
if is_str_list(val):
pass
else:
expr = val
""");
}
public void testTypeGuardListNegation() {
doTest("list[str]",
"""
from typing import List
from typing import TypeGuard
def is_str_list(val: List[object]) -> TypeGuard[List[str]]:
return all(isinstance(x, str) for x in val)
def func1(val: List[object]):
if not is_str_list(val):
pass
else:
expr = val
""");
}
// PY-62078
public void ignoreTestTypeGuardAnnotation() {
doTest("list[str]",
"""
from typing import List
from typing import TypeGuard
def is_str_list(val):
# type: (List[object]) -> TypeGuard[List[str]]
return all(isinstance(x, str) for x in val)
def func1(val: List[object]):
if not is_str_list(val):
pass
else:
expr = val
""");
}
public void testTypeGuardDidntChanged() {
doTest("list[object]",
"""
from typing import List
from typing import TypeGuard
def is_str_list(val: List[object]) -> TypeGuard[List[str]]:
return all(isinstance(x, str) for x in val)
def func1(val: List[object]):
if not is_str_list(val):
expr = val
else:
pass
""");
}
public void testTypeGuardDoubleCheck() {
doTest("Person",
"""
from typing import TypeGuard
class Person(TypedDict):
name: str
age: int
def is_person(val: dict) -> TypeGuard[Person]:
try:
return isinstance(val["name"], str) and isinstance(val["age"], int)
except KeyError:
return False
def print_age(val: dict, val2: dict):
if is_person(val) and is_person(val2):
expr = val
else:
print("Not a person!")""");
}
public void testTypeGuardDoubleCheckNegation() {
doTest("Person",
"""
from typing import TypeGuard
class Person(TypedDict):
name: str
age: int
def is_person(val: dict) -> TypeGuard[Person]:
try:
return isinstance(val["name"], str) and isinstance(val["age"], int)
except KeyError:
return False
def print_age(val: dict, val2: dict):
if not is_person(val) or not is_person(val2):
print("Not a person!");
else:
expr = val
""");
}
public void testDictCallOnDictLiteralResult() {
doTest("dict[LiteralString, int]",
"expr = dict({'a': 1})");
@@ -488,6 +488,18 @@ public class PyControlFlowBuilderTest extends LightMarkedTestCase {
doTest();
}
public void testTypeGuard() {
doTest();
}
public void testTypeGuardConjunct() {
doTest();
}
public void testTypeGuardWhile() {
doTest();
}
// PY-23859
public void testControlFlowIsAbruptAfterSelfFail() {
final String testName = getTestName(false);
@@ -228,6 +228,10 @@ public class PyUnreachableCodeInspectionTest extends PyInspectionTestCase {
doTest();
}
public void testUnreachableCodeReportedAfterNoReturnWithQuotesFunction() {
doTest();
}
// PY-24273
public void testUnreachableCodeReportedAfterImportedNoReturnFunction() {
doMultiFileTest();