From 2d1cef87ec90f939b1879b9ea96ac42e1000dd80 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 18 Aug 2014 20:50:27 +0400 Subject: [PATCH 01/22] Extracted QualifiedNameFinder.getQualifiedName() from PyQualifiedNameOwner implementations --- .../python/psi/impl/PyClassImpl.java | 23 +-------------- .../python/psi/impl/PyFunctionImpl.java | 17 +---------- .../psi/resolve/QualifiedNameFinder.java | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java b/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java index 7538fbae2fd2..fad945b9ef3d 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java @@ -20,7 +20,6 @@ import com.intellij.lang.ASTNode; import com.intellij.openapi.util.Comparing; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.NotNullLazyValue; -import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.*; import com.intellij.psi.scope.PsiScopeProcessor; import com.intellij.psi.search.LocalSearchScope; @@ -237,27 +236,7 @@ public class PyClassImpl extends PyPresentableElementImpl implement @Nullable public String getQualifiedName() { - String name = getName(); - final PyClassStub stub = getStub(); - PsiElement ancestor = stub != null ? stub.getParentStub().getPsi() : getParent(); - while (!(ancestor instanceof PsiFile)) { - if (ancestor == null) return name; // can this happen? - if (ancestor instanceof PyClass) { - name = ((PyClass)ancestor).getName() + "." + name; - } - ancestor = stub != null ? ((StubBasedPsiElement)ancestor).getStub().getParentStub().getPsi() : ancestor.getParent(); - } - - PsiFile psiFile = ((PsiFile)ancestor).getOriginalFile(); - final PyFile builtins = PyBuiltinCache.getInstance(this).getBuiltinsFile(); - if (!psiFile.equals(builtins)) { - VirtualFile vFile = psiFile.getVirtualFile(); - if (vFile != null) { - final String packageName = QualifiedNameFinder.findShortestImportableName(this, vFile); - return packageName + "." + name; - } - } - return name; + return QualifiedNameFinder.getQualifiedName(this); } @Override diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index 2531b37bf8b2..246b58d963c3 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -699,21 +699,6 @@ public class PyFunctionImpl extends PyPresentableElementImpl imp @Nullable @Override public String getQualifiedName() { - String name = getName(); - if (name == null) { - return null; - } - PyClass containingClass = getContainingClass(); - if (containingClass != null) { - return containingClass.getQualifiedName() + "." + name; - } - if (PsiTreeUtil.getStubOrPsiParent(this) instanceof PyFile) { - VirtualFile virtualFile = getContainingFile().getVirtualFile(); - if (virtualFile != null) { - final String packageName = QualifiedNameFinder.findShortestImportableName(this, virtualFile); - return packageName + "." + name; - } - } - return null; + return QualifiedNameFinder.getQualifiedName(this); } } diff --git a/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java b/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java index 97a15da105c3..5dff29010e63 100644 --- a/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java +++ b/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java @@ -24,10 +24,14 @@ import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.*; import com.jetbrains.python.PyNames; +import com.jetbrains.python.codeInsight.controlflow.ScopeOwner; +import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil; import com.jetbrains.python.psi.PyClass; +import com.jetbrains.python.psi.PyElement; import com.jetbrains.python.psi.PyFile; import com.jetbrains.python.psi.PyFunction; import com.intellij.psi.util.QualifiedName; +import com.jetbrains.python.psi.impl.PyBuiltinCache; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -156,6 +160,31 @@ public class QualifiedNameFinder { return qname; } + @Nullable + public static String getQualifiedName(@NotNull PyElement element) { + final String name = element.getName(); + if (name != null) { + final ScopeOwner owner = ScopeUtil.getScopeOwner(element); + final PyBuiltinCache builtinCache = PyBuiltinCache.getInstance(element); + if (owner instanceof PyClass) { + final String classQName = ((PyClass)owner).getQualifiedName(); + if (classQName != null) { + return classQName + "." + name; + } + } + else if (owner instanceof PyFile && !builtinCache.isBuiltin(element)) { + final VirtualFile virtualFile = ((PyFile)owner).getVirtualFile(); + if (virtualFile != null) { + final String fileQName = findShortestImportableName(element, virtualFile); + if (fileQName != null) { + return fileQName + "." + name; + } + } + } + } + return null; + } + /** * Tries to find roots that contain given vfile, and among them the root that contains at the smallest depth. * For equal depth source root is in preference to library. From 7228288e91e0d8dea1746e2d181c6be0dd39c183 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 18 Aug 2014 20:51:07 +0400 Subject: [PATCH 02/22] Made PyTargetExpression a PyQualifiedNameOwner --- .../src/com/jetbrains/python/psi/PyTargetExpression.java | 2 +- .../jetbrains/python/psi/impl/PyTargetExpressionImpl.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyTargetExpression.java b/python/psi-api/src/com/jetbrains/python/psi/PyTargetExpression.java index 0b67d85d3dc3..399b8f8a7bac 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyTargetExpression.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyTargetExpression.java @@ -28,7 +28,7 @@ import org.jetbrains.annotations.Nullable; * @author yole */ public interface PyTargetExpression extends PyQualifiedExpression, PsiNamedElement, PsiNameIdentifierOwner, PyDocStringOwner, - StubBasedPsiElement { + PyQualifiedNameOwner, StubBasedPsiElement { PyTargetExpression[] EMPTY_ARRAY = new PyTargetExpression[0]; /** diff --git a/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java index 545e8e86cba1..3805c8df525e 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java @@ -43,6 +43,7 @@ import com.jetbrains.python.psi.impl.references.PyQualifiedReference; import com.jetbrains.python.psi.impl.references.PyTargetReference; import com.jetbrains.python.psi.impl.stubs.CustomTargetExpressionStub; import com.jetbrains.python.psi.resolve.PyResolveContext; +import com.jetbrains.python.psi.resolve.QualifiedNameFinder; import com.jetbrains.python.psi.resolve.RatedResolveResult; import com.jetbrains.python.psi.stubs.PyClassStub; import com.jetbrains.python.psi.stubs.PyFunctionStub; @@ -707,4 +708,10 @@ public class PyTargetExpressionImpl extends PyPresentableElementImpl Date: Mon, 18 Aug 2014 21:51:11 +0400 Subject: [PATCH 03/22] Fixed QualifiedNameFinder.getQualifiedName() for built-in names --- .../python/psi/resolve/QualifiedNameFinder.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java b/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java index 5dff29010e63..1c081eff9356 100644 --- a/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java +++ b/python/src/com/jetbrains/python/psi/resolve/QualifiedNameFinder.java @@ -172,12 +172,17 @@ public class QualifiedNameFinder { return classQName + "." + name; } } - else if (owner instanceof PyFile && !builtinCache.isBuiltin(element)) { - final VirtualFile virtualFile = ((PyFile)owner).getVirtualFile(); - if (virtualFile != null) { - final String fileQName = findShortestImportableName(element, virtualFile); - if (fileQName != null) { - return fileQName + "." + name; + else if (owner instanceof PyFile) { + if (builtinCache.isBuiltin(element)) { + return name; + } + else { + final VirtualFile virtualFile = ((PyFile)owner).getVirtualFile(); + if (virtualFile != null) { + final String fileQName = findShortestImportableName(element, virtualFile); + if (fileQName != null) { + return fileQName + "." + name; + } } } } From 0968f995ea3f54bf018976a58afc9183e112a2ef Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 18 Aug 2014 22:44:04 +0400 Subject: [PATCH 04/22] Initial tests for types defined by mypy's typing module --- python/testData/typing/typing.py | 0 .../com/jetbrains/python/PyTypingTest.java | 80 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 python/testData/typing/typing.py create mode 100644 python/testSrc/com/jetbrains/python/PyTypingTest.java diff --git a/python/testData/typing/typing.py b/python/testData/typing/typing.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java new file mode 100644 index 000000000000..b8f661e67ca6 --- /dev/null +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python; + +import com.intellij.testFramework.LightProjectDescriptor; +import com.jetbrains.python.documentation.PythonDocumentationProvider; +import com.jetbrains.python.fixtures.PyTestCase; +import com.jetbrains.python.psi.LanguageLevel; +import com.jetbrains.python.psi.PyExpression; +import com.jetbrains.python.psi.types.PyType; +import com.jetbrains.python.psi.types.TypeEvalContext; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * @author vlan + */ +public class PyTypingTest extends PyTestCase { + @Nullable + @Override + protected LightProjectDescriptor getProjectDescriptor() { + return ourPy3Descriptor; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + setLanguageLevel(LanguageLevel.PYTHON32); + } + + @Override + public void tearDown() throws Exception { + setLanguageLevel(null); + super.tearDown(); + } + + public void testClassType() { + doTest("Foo", + "class Foo:" + + " pass\n" + + "\n" + + "def f(expr: Foo):\n" + + " pass\n"); + } + + public void testClassReturnType() { + doTest("Foo", + "class Foo:" + + " pass\n" + + "\n" + + "def f() -> Foo:\n" + + " pass\n" + + "\n" + + "expr = f()\n"); + + } + + private void doTest(@NotNull String expectedType, @NotNull String text) { + myFixture.copyDirectoryToProject("typing", ""); + myFixture.configureByText(PythonFileType.INSTANCE, text); + final PyExpression expr = myFixture.findElementByText("expr", PyExpression.class); + final TypeEvalContext context = TypeEvalContext.userInitiated(expr.getContainingFile()).withTracing(); + PyType actual = context.getType(expr); + final String actualType = PythonDocumentationProvider.getTypeName(actual, context); + assertEquals(expectedType, actualType); + } +} From 184e12ec193865f8acb314f4515168d43c8c2d30 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Mon, 18 Aug 2014 23:49:12 +0400 Subject: [PATCH 05/22] Added nullity annotations --- python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java | 1 + python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java b/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java index a6182fb2eb65..cb37c46cac68 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java @@ -23,6 +23,7 @@ import org.jetbrains.annotations.Nullable; * @author yole */ public interface PyAnnotation extends PyElement, StubBasedPsiElement { + @Nullable PyExpression getValue(); @Nullable diff --git a/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java b/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java index f3c806dac7c5..2a95c2bdcbf0 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java @@ -24,6 +24,7 @@ import com.jetbrains.python.psi.PyClass; import com.jetbrains.python.psi.PyExpression; import com.jetbrains.python.psi.PyReferenceExpression; import com.jetbrains.python.psi.stubs.PyAnnotationStub; +import org.jetbrains.annotations.Nullable; /** * @author yole @@ -37,11 +38,13 @@ public class PyAnnotationImpl extends PyBaseElementImpl implem super(stub, PyElementTypes.ANNOTATION); } + @Nullable @Override public PyExpression getValue() { return findChildByClass(PyExpression.class); } + @Nullable @Override public PyClass resolveToClass() { PyExpression expr = getValue(); From b8d75dc401a9ddc03096c92f0c4a611a219e6d85 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 19 Aug 2014 00:14:18 +0400 Subject: [PATCH 06/22] Refactored PyAnnotation into a PyTypedElement --- .../jetbrains/python/psi/PyAnnotation.java | 5 +--- .../python/psi/impl/PyAnnotationImpl.java | 24 ++++++++++--------- .../python/psi/impl/PyFunctionImpl.java | 10 ++++---- .../python/psi/impl/PyNamedParameterImpl.java | 10 ++++---- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java b/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java index cb37c46cac68..71dc7286d84b 100644 --- a/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java +++ b/python/psi-api/src/com/jetbrains/python/psi/PyAnnotation.java @@ -22,10 +22,7 @@ import org.jetbrains.annotations.Nullable; /** * @author yole */ -public interface PyAnnotation extends PyElement, StubBasedPsiElement { +public interface PyAnnotation extends PyTypedElement, StubBasedPsiElement { @Nullable PyExpression getValue(); - - @Nullable - PyClass resolveToClass(); } diff --git a/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java b/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java index 2a95c2bdcbf0..79a226c64eae 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java @@ -16,14 +16,14 @@ package com.jetbrains.python.psi.impl; import com.intellij.lang.ASTNode; -import com.intellij.psi.PsiElement; -import com.intellij.psi.PsiPolyVariantReference; import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.psi.PyAnnotation; -import com.jetbrains.python.psi.PyClass; import com.jetbrains.python.psi.PyExpression; -import com.jetbrains.python.psi.PyReferenceExpression; import com.jetbrains.python.psi.stubs.PyAnnotationStub; +import com.jetbrains.python.psi.types.PyClassLikeType; +import com.jetbrains.python.psi.types.PyType; +import com.jetbrains.python.psi.types.TypeEvalContext; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** @@ -46,13 +46,15 @@ public class PyAnnotationImpl extends PyBaseElementImpl implem @Nullable @Override - public PyClass resolveToClass() { - PyExpression expr = getValue(); - if (expr instanceof PyReferenceExpression) { - final PsiPolyVariantReference reference = ((PyReferenceExpression)expr).getReference(); - final PsiElement result = reference.resolve(); - if (result instanceof PyClass) { - return (PyClass) result; + public PyType getType(@NotNull TypeEvalContext context, @NotNull TypeEvalContext.Key key) { + final PyExpression value = getValue(); + if (value != null) { + final PyType type = context.getType(value); + if (type instanceof PyClassLikeType) { + final PyClassLikeType classType = (PyClassLikeType)type; + if (classType.isDefinition()) { + return classType.toInstance(); + } } } return null; diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index 246b58d963c3..d7c29fe4a830 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -184,11 +184,11 @@ public class PyFunctionImpl extends PyPresentableElementImpl imp } } if (context.maySwitchToAST(this) && LanguageLevel.forElement(this).isAtLeast(LanguageLevel.PYTHON30)) { - PyAnnotation anno = getAnnotation(); - if (anno != null) { - PyClass pyClass = anno.resolveToClass(); - if (pyClass != null) { - return new PyClassTypeImpl(pyClass, false); + final PyAnnotation annotation = getAnnotation(); + if (annotation != null) { + final PyType type = context.getType(annotation); + if (type != null) { + return type; } } } diff --git a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java index bf792fef0dae..cc9178c07f9e 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyNamedParameterImpl.java @@ -181,11 +181,11 @@ public class PyNamedParameterImpl extends PyPresentableElementImpl Date: Tue, 19 Aug 2014 00:14:59 +0400 Subject: [PATCH 07/22] Allow 'None' as a type defined by an annotation --- .../jetbrains/python/psi/impl/PyAnnotationImpl.java | 4 ++++ .../testSrc/com/jetbrains/python/PyTypingTest.java | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java b/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java index 79a226c64eae..f8a4d482c2f2 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyAnnotationImpl.java @@ -21,6 +21,7 @@ import com.jetbrains.python.psi.PyAnnotation; import com.jetbrains.python.psi.PyExpression; import com.jetbrains.python.psi.stubs.PyAnnotationStub; import com.jetbrains.python.psi.types.PyClassLikeType; +import com.jetbrains.python.psi.types.PyNoneType; import com.jetbrains.python.psi.types.PyType; import com.jetbrains.python.psi.types.TypeEvalContext; import org.jetbrains.annotations.NotNull; @@ -56,6 +57,9 @@ public class PyAnnotationImpl extends PyBaseElementImpl implem return classType.toInstance(); } } + else if (type instanceof PyNoneType) { + return type; + } } return null; } diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index b8f661e67ca6..7efc34498a32 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -68,6 +68,19 @@ public class PyTypingTest extends PyTestCase { } + public void testNoneType() { + doTest("None", + "def f(expr: None):\n" + + " pass\n"); + } + + public void testNoneReturnType() { + doTest("None", + "def f() -> None:\n" + + " return 0\n" + + "expr = f()\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From b85717541fc0eecf865431f36ef45f0f25fb7fbe Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 19 Aug 2014 16:46:33 +0400 Subject: [PATCH 08/22] Support for typing.Union type --- python/src/META-INF/python-core.xml | 3 + .../codeInsight/PyTypingTypeProvider.java | 116 ++++ python/testData/typing/typing.py | 582 ++++++++++++++++++ .../com/jetbrains/python/PyTypingTest.java | 22 +- 4 files changed, 720 insertions(+), 3 deletions(-) create mode 100644 python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java diff --git a/python/src/META-INF/python-core.xml b/python/src/META-INF/python-core.xml index 292237aac6ac..7adbc46c189c 100644 --- a/python/src/META-INF/python-core.xml +++ b/python/src/META-INF/python-core.xml @@ -589,6 +589,9 @@ + + + diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java new file mode 100644 index 000000000000..7ee234b546cf --- /dev/null +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -0,0 +1,116 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.codeInsight; + +import com.intellij.psi.PsiElement; +import com.intellij.psi.PsiPolyVariantReference; +import com.jetbrains.python.psi.*; +import com.jetbrains.python.psi.resolve.PyResolveContext; +import com.jetbrains.python.psi.types.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author vlan + */ +public class PyTypingTypeProvider extends PyTypeProviderBase { + @Override + public PyType getParameterType(@NotNull PyNamedParameter param, @NotNull PyFunction func, @NotNull TypeEvalContext context) { + final PyAnnotation annotation = param.getAnnotation(); + if (annotation != null) { + final PyExpression value = annotation.getValue(); + if (value != null) { + return getTypingType(value, context); + } + } + return null; + } + + @Nullable + @Override + public PyType getReturnType(@NotNull Callable callable, @NotNull TypeEvalContext context) { + if (callable instanceof PyFunction) { + final PyFunction function = (PyFunction)callable; + final PyAnnotation annotation = function.getAnnotation(); + if (annotation != null) { + final PyExpression value = annotation.getValue(); + if (value != null) { + return getTypingType(value, context); + } + } + } + return null; + } + + @Nullable + private static PyType getTypingType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + // TODO: Put the annotation text into stubs and parse it to avoid switching from stubs to AST + if (expression instanceof PySubscriptionExpression) { + final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; + final PyExpression operand = subscriptionExpr.getOperand(); + final String operandName = resolveToQualifiedName(operand, context); + if ("typing.Union".equals(operandName)) { + final PyExpression indexExpr = subscriptionExpr.getIndexExpression(); + if (indexExpr instanceof PyTupleExpression) { + final PyTupleExpression tupleExpr = (PyTupleExpression)indexExpr; + final List types = new ArrayList(); + for (PyExpression expr : tupleExpr.getElements()) { + types.add(getType(expr, context)); + } + return PyUnionType.union(types); + } + } + } + return null; + } + + @Nullable + private static PyType getType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + final PyType typingType = getTypingType(expression, context); + if (typingType != null) { + return typingType; + } + final PyType type = context.getType(expression); + if (type instanceof PyClassLikeType) { + final PyClassLikeType classType = (PyClassLikeType)type; + if (classType.isDefinition()) { + return classType.toInstance(); + } + } + else if (type instanceof PyNoneType) { + return type; + } + return null; + } + + @Nullable + private static String resolveToQualifiedName(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + if (expression instanceof PyReferenceOwner) { + final PyReferenceOwner referenceOwner = (PyReferenceOwner)expression; + final PyResolveContext resolveContext = PyResolveContext.noImplicits().withTypeEvalContext(context); + final PsiPolyVariantReference reference = referenceOwner.getReference(resolveContext); + final PsiElement element = reference.resolve(); + if (element instanceof PyQualifiedNameOwner) { + final PyQualifiedNameOwner qualifiedNameOwner = (PyQualifiedNameOwner)element; + return qualifiedNameOwner.getQualifiedName(); + } + } + return null; + } +} diff --git a/python/testData/typing/typing.py b/python/testData/typing/typing.py index e69de29bb2d1..1fbd5fc8081b 100644 --- a/python/testData/typing/typing.py +++ b/python/testData/typing/typing.py @@ -0,0 +1,582 @@ +"""Static type checking helpers""" + +from abc import ABCMeta, abstractmethod, abstractproperty +import inspect +import sys +import re + + +__all__ = [ + # Type system related + 'AbstractGeneric', + 'AbstractGenericMeta', + 'Any', + 'AnyStr', + 'Dict', + 'Function', + 'Generic', + 'GenericMeta', + 'IO', + 'List', + 'Match', + 'Pattern', + 'Protocol', + 'Set', + 'Tuple', + 'Undefined', + 'Union', + 'cast', + 'forwardref', + 'overload', + 'typevar', + # Protocols and abstract base classes + 'Container', + 'Iterable', + 'Iterator', + 'Sequence', + 'Sized', + 'AbstractSet', + 'Mapping', + 'BinaryIO', + 'TextIO', +] + + +def builtinclass(cls): + """Mark a class as a built-in/extension class for type checking.""" + return cls + + +def ducktype(type): + """Return a duck type declaration decorator. + + The decorator only affects type checking. + """ + def decorator(cls): + return cls + return decorator + + +def disjointclass(type): + """Return a disjoint class declaration decorator. + + The decorator only affects type checking. + """ + def decorator(cls): + return cls + return decorator + + +class GenericMeta(type): + """Metaclass for generic classes that support indexing by types.""" + + def __getitem__(self, args): + # Just ignore args; they are for compile-time checks only. + return self + + +class Generic(metaclass=GenericMeta): + """Base class for generic classes.""" + + +class AbstractGenericMeta(ABCMeta): + """Metaclass for abstract generic classes that support type indexing. + + This is used for both protocols and ordinary abstract classes. + """ + + def __new__(mcls, name, bases, namespace): + cls = super().__new__(mcls, name, bases, namespace) + # 'Protocol' must be an explicit base class in order for a class to + # be a protocol. + cls._is_protocol = name == 'Protocol' or Protocol in bases + return cls + + def __getitem__(self, args): + # Just ignore args; they are for compile-time checks only. + return self + + +class Protocol(metaclass=AbstractGenericMeta): + """Base class for protocol classes.""" + + @classmethod + def __subclasshook__(cls, c): + if not cls._is_protocol: + # No structural checks since this isn't a protocol. + return NotImplemented + + if cls is Protocol: + # Every class is a subclass of the empty protocol. + return True + + # Find all attributes defined in the protocol. + attrs = cls._get_protocol_attrs() + + for attr in attrs: + if not any(attr in d.__dict__ for d in c.__mro__): + return NotImplemented + return True + + @classmethod + def _get_protocol_attrs(cls): + # Get all Protocol base classes. + protocol_bases = [] + for c in cls.__mro__: + if getattr(c, '_is_protocol', False) and c.__name__ != 'Protocol': + protocol_bases.append(c) + + # Get attributes included in protocol. + attrs = set() + for base in protocol_bases: + for attr in base.__dict__.keys(): + # Include attributes not defined in any non-protocol bases. + for c in cls.__mro__: + if (c is not base and attr in c.__dict__ and + not getattr(c, '_is_protocol', False)): + break + else: + if (not attr.startswith('_abc_') and + attr != '__abstractmethods__' and + attr != '_is_protocol' and + attr != '__dict__' and + attr != '_get_protocol_attrs' and + attr != '__module__'): + attrs.add(attr) + + return attrs + + +class AbstractGeneric(metaclass=AbstractGenericMeta): + """Base class for abstract generic classes.""" + + +class TypeAlias: + """Class for defining generic aliases for library types.""" + + def __init__(self, target_type): + self.target_type = target_type + + def __getitem__(self, typeargs): + return self.target_type + + +Traceback = object() # TODO proper type object + + +# Define aliases for built-in types that support indexing. +List = TypeAlias(list) +Dict = TypeAlias(dict) +Set = TypeAlias(set) +Tuple = TypeAlias(tuple) +Function = TypeAlias(callable) +Pattern = TypeAlias(type(re.compile(''))) +Match = TypeAlias(type(re.match('', ''))) + +def union(x): return x + +Union = TypeAlias(union) + +class typevar: + def __init__(self, name, *, values=None): + self.name = name + self.values = values + + +# Predefined type variables. +AnyStr = typevar('AnyStr', values=(str, bytes)) + + +class forwardref: + def __init__(self, name): + self.name = name + + +def Any(x): + """The Any type; can also be used to cast a value to type Any.""" + return x + +def cast(type, object): + """Cast a value to a type. + + This only affects static checking; simply return object at runtime. + """ + return object + + +def overload(func): + """Function decorator for defining overloaded functions.""" + frame = sys._getframe(1) + locals = frame.f_locals + # See if there is a previous overload variant available. Also verify + # that the existing function really is overloaded: otherwise, replace + # the definition. The latter is actually important if we want to reload + # a library module such as genericpath with a custom one that uses + # overloading in the implementation. + if func.__name__ in locals and hasattr(locals[func.__name__], 'dispatch'): + orig_func = locals[func.__name__] + + def wrapper(*args, **kwargs): + ret, ok = orig_func.dispatch(*args, **kwargs) + if ok: + return ret + return func(*args, **kwargs) + wrapper.isoverload = True + wrapper.dispatch = make_dispatcher(func, orig_func.dispatch) + wrapper.next = orig_func + wrapper.__name__ = func.__name__ + if hasattr(func, '__isabstractmethod__'): + # Note that we can't reliably check that abstractmethod is + # used consistently across overload variants, so we let a + # static checker do it. + wrapper.__isabstractmethod__ = func.__isabstractmethod__ + return wrapper + else: + # Return the initial overload variant. + func.isoverload = True + func.dispatch = make_dispatcher(func) + func.next = None + return func + + +def is_erased_type(t): + return t is Any or isinstance(t, typevar) + + +def make_dispatcher(func, previous=None): + """Create argument dispatcher for an overloaded function. + + Also handle chaining of multiple overload variants. + """ + (args, varargs, varkw, defaults, + kwonlyargs, kwonlydefaults, annotations) = inspect.getfullargspec(func) + + argtypes = [] + for arg in args: + ann = annotations.get(arg) + if isinstance(ann, forwardref): + ann = ann.name + if is_erased_type(ann): + ann = None + elif isinstance(ann, str): + # The annotation is a string => evaluate it lazily when the + # overloaded function is first called. + frame = sys._getframe(2) + t = [None] + ann_str = ann + def check(x): + if not t[0]: + # Evaluate string in the context of the overload caller. + t[0] = eval(ann_str, frame.f_globals, frame.f_locals) + if is_erased_type(t[0]): + # Anything goes. + t[0] = object + if isinstance(t[0], type): + return isinstance(x, t[0]) + else: + return t[0](x) + ann = check + argtypes.append(ann) + + maxargs = len(argtypes) + minargs = maxargs + if defaults: + minargs = len(argtypes) - len(defaults) + + def dispatch(*args, **kwargs): + if previous: + ret, ok = previous(*args, **kwargs) + if ok: + return ret, ok + + nargs = len(args) + if nargs < minargs or nargs > maxargs: + # Invalid argument count. + return None, False + + for i in range(nargs): + argtype = argtypes[i] + if argtype: + if isinstance(argtype, type): + if not isinstance(args[i], argtype): + break + else: + if not argtype(args[i]): + break + else: + return func(*args, **kwargs), True + return None, False + return dispatch + + +class Undefined: + """Class that represents an undefined value with a specified type. + + At runtime the name Undefined is bound to an instance of this + class. The intent is that any operation on an Undefined object + raises an exception, including use in a boolean context. Some + operations cannot be disallowed: Undefined can be used as an + operand of 'is', and it can be assigned to variables and stored in + containers. + + 'Undefined' makes it possible to declare the static type of a + variable even if there is no useful default value to initialize it + with: + + from typing import Undefined + x = Undefined(int) + y = Undefined # type: int + + The latter form can be used if efficiency is of utmost importance, + since it saves a call operation and potentially additional + operations needed to evaluate a type expression. Undefined(x) + just evaluates to Undefined, ignoring the argument value. + """ + + def __repr__(self): + return '' + + def __setattr__(self, attr, value): + raise AttributeError("'Undefined' object has no attribute '%s'" % attr) + + def __eq__(self, other): + raise TypeError("'Undefined' object cannot be compared") + + def __call__(self, type): + return self + + def __bool__(self): + raise TypeError("'Undefined' object is not valid as a boolean") + + +Undefined = Undefined() + + +# Abstract classes + + +T = typevar('T') +KT = typevar('KT') +VT = typevar('VT') + + +class SupportsInt(Protocol): + @abstractmethod + def __int__(self) -> int: pass + + +class SupportsFloat(Protocol): + @abstractmethod + def __float__(self) -> float: pass + + +class SupportsAbs(Protocol[T]): + @abstractmethod + def __abs__(self) -> T: pass + + +class SupportsRound(Protocol[T]): + @abstractmethod + def __round__(self, ndigits: int = 0) -> T: pass + + +class Reversible(Protocol[T]): + @abstractmethod + def __reversed__(self) -> 'Iterator[T]': pass + + +class Sized(Protocol): + @abstractmethod + def __len__(self) -> int: pass + + +class Container(Protocol[T]): + @abstractmethod + def __contains__(self, x) -> bool: pass + + +class Iterable(Protocol[T]): + @abstractmethod + def __iter__(self) -> 'Iterator[T]': pass + + +class Iterator(Iterable[T], Protocol[T]): + @abstractmethod + def __next__(self) -> T: pass + + +class Sequence(Sized, Iterable[T], Container[T], AbstractGeneric[T]): + @abstractmethod + @overload + def __getitem__(self, i: int) -> T: pass + + @abstractmethod + @overload + def __getitem__(self, s: slice) -> 'Sequence[T]': pass + + @abstractmethod + def __reversed__(self, s: slice) -> Iterator[T]: pass + + @abstractmethod + def index(self, x) -> int: pass + + @abstractmethod + def count(self, x) -> int: pass + + +for t in list, tuple, str, bytes, range: + Sequence.register(t) + + +class AbstractSet(Sized, Iterable[T], AbstractGeneric[T]): + @abstractmethod + def __contains__(self, x: object) -> bool: pass + @abstractmethod + def __and__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass + @abstractmethod + def __or__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass + @abstractmethod + def __sub__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass + @abstractmethod + def __xor__(self, s: 'AbstractSet[T]') -> 'AbstractSet[T]': pass + @abstractmethod + def isdisjoint(self, s: 'AbstractSet[T]') -> bool: pass + + +for t in set, frozenset, type({}.keys()), type({}.items()): + AbstractSet.register(t) + + +class Mapping(Sized, Iterable[KT], AbstractGeneric[KT, VT]): + @abstractmethod + def __getitem__(self, k: KT) -> VT: pass + @abstractmethod + def __setitem__(self, k: KT, v: VT) -> None: pass + @abstractmethod + def __delitem__(self, v: KT) -> None: pass + @abstractmethod + def __contains__(self, o: object) -> bool: pass + + @abstractmethod + def clear(self) -> None: pass + @abstractmethod + def copy(self) -> 'Mapping[KT, VT]': pass + @overload + @abstractmethod + def get(self, k: KT) -> VT: pass + @overload + @abstractmethod + def get(self, k: KT, default: VT) -> VT: pass + @overload + @abstractmethod + def pop(self, k: KT) -> VT: pass + @overload + @abstractmethod + def pop(self, k: KT, default: VT) -> VT: pass + @abstractmethod + def popitem(self) -> Tuple[KT, VT]: pass + @overload + @abstractmethod + def setdefault(self, k: KT) -> VT: pass + @overload + @abstractmethod + def setdefault(self, k: KT, default: VT) -> VT: pass + + @overload + @abstractmethod + def update(self, m: 'Mapping[KT, VT]') -> None: pass + @overload + @abstractmethod + def update(self, m: Iterable[Tuple[KT, VT]]) -> None: pass + + @abstractmethod + def keys(self) -> AbstractSet[KT]: pass + @abstractmethod + def values(self) -> AbstractSet[VT]: pass + @abstractmethod + def items(self) -> AbstractSet[Tuple[KT, VT]]: pass + + +# TODO Consider more types: os.environ, etc. However, these add dependencies. +Mapping.register(dict) + + +# Note that the BinaryIO and TextIO classes must be in sync with typing module +# stubs. + + +class IO(AbstractGeneric[AnyStr]): + @abstractproperty + def mode(self) -> str: pass + @abstractproperty + def name(self) -> str: pass + @abstractmethod + def close(self) -> None: pass + @abstractmethod + def closed(self) -> bool: pass + @abstractmethod + def fileno(self) -> int: pass + @abstractmethod + def flush(self) -> None: pass + @abstractmethod + def isatty(self) -> bool: pass + @abstractmethod + def read(self, n: int = -1) -> AnyStr: pass + @abstractmethod + def readable(self) -> bool: pass + @abstractmethod + def readline(self, limit: int = -1) -> AnyStr: pass + @abstractmethod + def readlines(self, hint: int = -1) -> List[AnyStr]: pass + @abstractmethod + def seek(self, offset: int, whence: int = 0) -> int: pass + @abstractmethod + def seekable(self) -> bool: pass + @abstractmethod + def tell(self) -> int: pass + @abstractmethod + def truncate(self, size: int = None) -> int: pass + @abstractmethod + def writable(self) -> bool: pass + @abstractmethod + def write(self, s: AnyStr) -> int: pass + @abstractmethod + def writelines(self, lines: List[AnyStr]) -> None: pass + + @abstractmethod + def __enter__(self) -> 'IO[AnyStr]': pass + @abstractmethod + def __exit__(self, type, value, traceback) -> None: pass + + +class BinaryIO(IO[bytes]): + @overload + @abstractmethod + def write(self, s: bytes) -> int: pass + @overload + @abstractmethod + def write(self, s: bytearray) -> int: pass + + @abstractmethod + def __enter__(self) -> 'BinaryIO': pass + + +class TextIO(IO[str]): + @abstractproperty + def buffer(self) -> BinaryIO: pass + @abstractproperty + def encoding(self) -> str: pass + @abstractproperty + def errors(self) -> str: pass + @abstractproperty + def line_buffering(self) -> bool: pass + @abstractproperty + def newlines(self) -> Any: pass + @abstractmethod + def __enter__(self) -> 'TextIO': pass + + +# TODO Register IO/TextIO/BinaryIO as the base class of file-like types. + + +del t diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index 7efc34498a32..ef8a0928dbd5 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -26,6 +26,8 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** + * Tests for a type system based on mypy's typing module. + * * @author vlan */ public class PyTypingTest extends PyTestCase { @@ -81,13 +83,27 @@ public class PyTypingTest extends PyTestCase { "expr = f()\n"); } + public void testUnionType() { + doTest("int | str", + "from typing import Union\n" + + "\n" + + "def f(expr: Union[int, str]):\n" + + " pass\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); final PyExpression expr = myFixture.findElementByText("expr", PyExpression.class); - final TypeEvalContext context = TypeEvalContext.userInitiated(expr.getContainingFile()).withTracing(); - PyType actual = context.getType(expr); + final TypeEvalContext codeAnalysis = TypeEvalContext.codeAnalysis(expr.getContainingFile()); + final TypeEvalContext userInitiated = TypeEvalContext.userInitiated(expr.getContainingFile()).withTracing(); + assertType(expectedType, expr, codeAnalysis, "code analysis"); + assertType(expectedType, expr, userInitiated, "user initiated"); + } + + private static void assertType(String expectedType, PyExpression expr, TypeEvalContext context, String contextName) { + final PyType actual = context.getType(expr); final String actualType = PythonDocumentationProvider.getTypeName(actual, context); - assertEquals(expectedType, actualType); + assertEquals("Failed in " + contextName + " context", expectedType, actualType); } } From 1f88c4dcd3a622d0212b807feb026a70700d2558 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 19 Aug 2014 18:57:41 +0400 Subject: [PATCH 09/22] Support for built-in collection types --- .../codeInsight/PyTypingTypeProvider.java | 58 ++++++++++++++++--- .../com/jetbrains/python/PyTypingTest.java | 48 +++++++++++++++ 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index 7ee234b546cf..cf90fe3b8ee4 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -15,8 +15,10 @@ */ package com.jetbrains.python.codeInsight; +import com.google.common.collect.ImmutableMap; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; +import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.types.*; @@ -30,7 +32,13 @@ import java.util.List; * @author vlan */ public class PyTypingTypeProvider extends PyTypeProviderBase { - @Override + private static ImmutableMap BUILTIN_COLLECTIONS = ImmutableMap.builder() + .put("typing.List", "list") + .put("typing.Dict", "dict") + .put("typing.Set", PyNames.SET) + .put("typing.Tuple", PyNames.TUPLE) + .build(); + public PyType getParameterType(@NotNull PyNamedParameter param, @NotNull PyFunction func, @NotNull TypeEvalContext context) { final PyAnnotation annotation = param.getAnnotation(); if (annotation != null) { @@ -64,24 +72,58 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { if (expression instanceof PySubscriptionExpression) { final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; final PyExpression operand = subscriptionExpr.getOperand(); + final PyExpression indexExpr = subscriptionExpr.getIndexExpression(); final String operandName = resolveToQualifiedName(operand, context); if ("typing.Union".equals(operandName)) { - final PyExpression indexExpr = subscriptionExpr.getIndexExpression(); - if (indexExpr instanceof PyTupleExpression) { - final PyTupleExpression tupleExpr = (PyTupleExpression)indexExpr; - final List types = new ArrayList(); - for (PyExpression expr : tupleExpr.getElements()) { - types.add(getType(expr, context)); + return PyUnionType.union(getIndexTypes(subscriptionExpr, context)); + } + else { + final PyType operandType = getType(operand, context); + if (operandType instanceof PyClassType) { + final PyClass cls = ((PyClassType)operandType).getPyClass(); + if (PyNames.TUPLE.equals(cls.getQualifiedName())) { + final List indexTypes = getIndexTypes(subscriptionExpr, context); + return PyTupleType.create(expression, indexTypes.toArray(new PyType[indexTypes.size()])); + } + else if (indexExpr != null) { + final PyType indexType = context.getType(indexExpr); + return new PyCollectionTypeImpl(cls, false, indexType); } - return PyUnionType.union(types); } } } + else { + final PyType builtinCollection = getBuiltinCollection(expression, context); + if (builtinCollection != null) { + return builtinCollection; + } + } return null; } + @NotNull + private static List getIndexTypes(@NotNull PySubscriptionExpression expression, @NotNull TypeEvalContext context) { + final List types = new ArrayList(); + final PyExpression indexExpr = expression.getIndexExpression(); + if (indexExpr instanceof PyTupleExpression) { + final PyTupleExpression tupleExpr = (PyTupleExpression)indexExpr; + for (PyExpression expr : tupleExpr.getElements()) { + types.add(getType(expr, context)); + } + } + return types; + } + + @Nullable + private static PyType getBuiltinCollection(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + final String collectionName = resolveToQualifiedName(expression, context); + final String builtinName = BUILTIN_COLLECTIONS.get(collectionName); + return builtinName != null ? PyTypeParser.getTypeByName(expression, builtinName) : null; + } + @Nullable private static PyType getType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + // It is possible to replace PyAnnotation.getType() with this implementation final PyType typingType = getTypingType(expression, context); if (typingType != null) { return typingType; diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index ef8a0928dbd5..b63e17bbbeee 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -91,6 +91,54 @@ public class PyTypingTest extends PyTestCase { " pass\n"); } + public void testBuiltinList() { + doTest("list", + "from typing import List\n" + + "\n" + + "def f(expr: List):\n" + + " pass\n"); + } + + public void testBuiltinListWithParameter() { + doTest("list[int]", + "from typing import List\n" + + "\n" + + "def f(expr: List[int]):\n" + + " pass\n"); + } + + public void testBuiltinDictWithParameters() { + doTest("dict[str, int]", + "from typing import Dict\n" + + "\n" + + "def f(expr: Dict[str, int]):\n" + + " pass\n"); + } + + public void testBuiltinTuple() { + doTest("tuple", + "from typing import Tuple\n" + + "\n" + + "def f(expr: Tuple):\n" + + " pass\n"); + } + + public void testBuiltinTupleWithParameters() { + doTest("(int, str)", + "from typing import Tuple\n" + + "\n" + + "def f(expr: Tuple[int, str]):\n" + + " pass\n"); + } + + public void testAnyType() { + doTest("unknown", + "from typing import Any\n" + + "\n" + + "def f(expr: Any):\n" + + " pass\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From 7685df3418ed20da7af96ce4e7c7c6331250def5 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Tue, 19 Aug 2014 18:58:51 +0400 Subject: [PATCH 10/22] Updated stubs to AST comments --- .../com/jetbrains/python/codeInsight/PyTypingTypeProvider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index cf90fe3b8ee4..f873f8d3b939 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -42,6 +42,7 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { public PyType getParameterType(@NotNull PyNamedParameter param, @NotNull PyFunction func, @NotNull TypeEvalContext context) { final PyAnnotation annotation = param.getAnnotation(); if (annotation != null) { + // XXX: Requires switching from stub to AST final PyExpression value = annotation.getValue(); if (value != null) { return getTypingType(value, context); @@ -57,6 +58,7 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { final PyFunction function = (PyFunction)callable; final PyAnnotation annotation = function.getAnnotation(); if (annotation != null) { + // XXX: Requires switching from stub to AST final PyExpression value = annotation.getValue(); if (value != null) { return getTypingType(value, context); @@ -68,7 +70,6 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { @Nullable private static PyType getTypingType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { - // TODO: Put the annotation text into stubs and parse it to avoid switching from stubs to AST if (expression instanceof PySubscriptionExpression) { final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; final PyExpression operand = subscriptionExpr.getOperand(); From 58bc1b9d33bc92c38d2c0c674ca484f960589e24 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 20 Aug 2014 16:32:29 +0400 Subject: [PATCH 11/22] Extracted resolve() for PyExpression --- .../codeInsight/PyTypingTypeProvider.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index f873f8d3b939..a2ce9b8b8f60 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -143,16 +143,22 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { } @Nullable - private static String resolveToQualifiedName(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + private static PsiElement resolve(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { if (expression instanceof PyReferenceOwner) { final PyReferenceOwner referenceOwner = (PyReferenceOwner)expression; final PyResolveContext resolveContext = PyResolveContext.noImplicits().withTypeEvalContext(context); final PsiPolyVariantReference reference = referenceOwner.getReference(resolveContext); - final PsiElement element = reference.resolve(); - if (element instanceof PyQualifiedNameOwner) { - final PyQualifiedNameOwner qualifiedNameOwner = (PyQualifiedNameOwner)element; - return qualifiedNameOwner.getQualifiedName(); - } + return reference.resolve(); + } + return null; + } + + @Nullable + private static String resolveToQualifiedName(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + final PsiElement element = resolve(expression, context); + if (element instanceof PyQualifiedNameOwner) { + final PyQualifiedNameOwner qualifiedNameOwner = (PyQualifiedNameOwner)element; + return qualifiedNameOwner.getQualifiedName(); } return null; } From e1e81814026c11269ca56ce3f8610f41fe9d6eb8 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 20 Aug 2014 17:15:15 +0400 Subject: [PATCH 12/22] Support for 'typevar' generic types without inheritance --- .../codeInsight/PyTypingTypeProvider.java | 71 ++++++++++++++++++- .../com/jetbrains/python/PyTypingTest.java | 21 ++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index a2ce9b8b8f60..ea573ebdd5e3 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -18,8 +18,10 @@ package com.jetbrains.python.codeInsight; import com.google.common.collect.ImmutableMap; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; +import com.intellij.psi.util.QualifiedName; import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; +import com.jetbrains.python.psi.impl.PyPsiUtils; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.types.*; import org.jetbrains.annotations.NotNull; @@ -98,10 +100,67 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { if (builtinCollection != null) { return builtinCollection; } + final PyType genericType = getGenericType(expression, context); + if (genericType != null) { + return genericType; + } } return null; } + @Nullable + private static PyType getGenericType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + final PsiElement resolved = resolve(expression, context); + if (resolved instanceof PyTargetExpression) { + final PyTargetExpression targetExpr = (PyTargetExpression)resolved; + final QualifiedName calleeName = targetExpr.getCalleeName(); + if (calleeName != null && "typevar".equals(calleeName.toString())) { + // XXX: Requires switching from stub to AST + final PyExpression assigned = targetExpr.findAssignedValue(); + if (assigned instanceof PyCallExpression) { + final PyCallExpression assignedCall = (PyCallExpression)assigned; + final PyExpression callee = assignedCall.getCallee(); + if (callee != null) { + final String calleeQName = resolveToQualifiedName(callee, context); + if ("typing.typevar".equals(calleeQName)) { + final PyExpression[] arguments = assignedCall.getArguments(); + if (arguments.length > 0) { + final PyExpression firstArgument = arguments[0]; + if (firstArgument instanceof PyStringLiteralExpression) { + final String name = ((PyStringLiteralExpression)firstArgument).getStringValue(); + if (name != null) { + return new PyGenericType(name, getGenericTypeBound(arguments, context)); + } + } + } + } + + } + } + } + } + return null; + } + + @Nullable + private static PyType getGenericTypeBound(@NotNull PyExpression[] typeVarArguments, @NotNull TypeEvalContext context) { + final List types = new ArrayList(); + if (typeVarArguments.length > 1) { + final PyExpression secondArgument = typeVarArguments[1]; + if (secondArgument instanceof PyKeywordArgument) { + final PyKeywordArgument valuesArgument = (PyKeywordArgument)secondArgument; + final PyExpression valueExpr = PyPsiUtils.flattenParens(valuesArgument.getValueExpression()); + if (valueExpr instanceof PyTupleExpression) { + final PyTupleExpression tupleExpr = (PyTupleExpression)valueExpr; + for (PyExpression expr : tupleExpr.getElements()) { + types.add(getType(expr, context)); + } + } + } + } + return PyUnionType.union(types); + } + @NotNull private static List getIndexTypes(@NotNull PySubscriptionExpression expression, @NotNull TypeEvalContext context) { final List types = new ArrayList(); @@ -148,7 +207,17 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { final PyReferenceOwner referenceOwner = (PyReferenceOwner)expression; final PyResolveContext resolveContext = PyResolveContext.noImplicits().withTypeEvalContext(context); final PsiPolyVariantReference reference = referenceOwner.getReference(resolveContext); - return reference.resolve(); + final PsiElement element = reference.resolve(); + if (element instanceof PyFunction) { + final PyFunction function = (PyFunction)element; + if (PyUtil.isInit(function)) { + final PyClass cls = function.getContainingClass(); + if (cls != null) { + return cls; + } + } + } + return element; } return null; } diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index b63e17bbbeee..afe0d094a527 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -139,6 +139,27 @@ public class PyTypingTest extends PyTestCase { " pass\n"); } + public void testGenericType() { + doTest("A", + "from typing import typevar" + + "\n" + + "T = typevar('A')\n" + + "\n" + + "def f(expr: T):\n" + + " pass\n"); + } + + public void testGenericBoundedType() { + doTest("T <= int | str", + "from typing import typevar" + + "\n" + + "T = typevar('T', values=(int, str))\n" + + "\n" + + "def f(expr: T):\n" + + " pass\n"); + + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From 88c5b7958ee08d8cba3ed8fd63efc96aa032bf9c Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 20 Aug 2014 17:21:30 +0400 Subject: [PATCH 13/22] Extracted getUnionType() and getParameterizedType() --- .../codeInsight/PyTypingTypeProvider.java | 68 ++++++++++++------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index ea573ebdd5e3..2e7c3f3bece3 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -72,38 +72,34 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { @Nullable private static PyType getTypingType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + final PyType unionType = getUnionType(expression, context); + if (unionType != null) { + return unionType; + } + final PyType parameterizedType = getParameterizedType(expression, context); + if (parameterizedType != null) { + return parameterizedType; + } + final PyType builtinCollection = getBuiltinCollection(expression, context); + if (builtinCollection != null) { + return builtinCollection; + } + final PyType genericType = getGenericType(expression, context); + if (genericType != null) { + return genericType; + } + return null; + } + + @Nullable + private static PyType getUnionType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { if (expression instanceof PySubscriptionExpression) { final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; final PyExpression operand = subscriptionExpr.getOperand(); - final PyExpression indexExpr = subscriptionExpr.getIndexExpression(); final String operandName = resolveToQualifiedName(operand, context); if ("typing.Union".equals(operandName)) { return PyUnionType.union(getIndexTypes(subscriptionExpr, context)); } - else { - final PyType operandType = getType(operand, context); - if (operandType instanceof PyClassType) { - final PyClass cls = ((PyClassType)operandType).getPyClass(); - if (PyNames.TUPLE.equals(cls.getQualifiedName())) { - final List indexTypes = getIndexTypes(subscriptionExpr, context); - return PyTupleType.create(expression, indexTypes.toArray(new PyType[indexTypes.size()])); - } - else if (indexExpr != null) { - final PyType indexType = context.getType(indexExpr); - return new PyCollectionTypeImpl(cls, false, indexType); - } - } - } - } - else { - final PyType builtinCollection = getBuiltinCollection(expression, context); - if (builtinCollection != null) { - return builtinCollection; - } - final PyType genericType = getGenericType(expression, context); - if (genericType != null) { - return genericType; - } } return null; } @@ -174,6 +170,28 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { return types; } + @Nullable + private static PyType getParameterizedType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + if (expression instanceof PySubscriptionExpression) { + final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; + final PyExpression operand = subscriptionExpr.getOperand(); + final PyExpression indexExpr = subscriptionExpr.getIndexExpression(); + final PyType operandType = getType(operand, context); + if (operandType instanceof PyClassType) { + final PyClass cls = ((PyClassType)operandType).getPyClass(); + if (PyNames.TUPLE.equals(cls.getQualifiedName())) { + final List indexTypes = getIndexTypes(subscriptionExpr, context); + return PyTupleType.create(expression, indexTypes.toArray(new PyType[indexTypes.size()])); + } + else if (indexExpr != null) { + final PyType indexType = context.getType(indexExpr); + return new PyCollectionTypeImpl(cls, false, indexType); + } + } + } + return null; + } + @Nullable private static PyType getBuiltinCollection(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { final String collectionName = resolveToQualifiedName(expression, context); From e04d3efb142ba371c05c0bbf5e8e34298bea22f5 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Wed, 20 Aug 2014 17:25:56 +0400 Subject: [PATCH 14/22] Missing newlines --- python/testSrc/com/jetbrains/python/PyTypingTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index afe0d094a527..d2841569041b 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -141,7 +141,7 @@ public class PyTypingTest extends PyTestCase { public void testGenericType() { doTest("A", - "from typing import typevar" + + "from typing import typevar\n" + "\n" + "T = typevar('A')\n" + "\n" + @@ -151,13 +151,12 @@ public class PyTypingTest extends PyTestCase { public void testGenericBoundedType() { doTest("T <= int | str", - "from typing import typevar" + + "from typing import typevar\n" + "\n" + "T = typevar('T', values=(int, str))\n" + "\n" + "def f(expr: T):\n" + " pass\n"); - } private void doTest(@NotNull String expectedType, @NotNull String text) { From ea3333f70c205177b49ed7a7b82f1fda8a4a322d Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 21 Aug 2014 14:49:56 +0400 Subject: [PATCH 15/22] Fixed searching for PyAnnotation stub if PyFunction is a stub --- python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java index d7c29fe4a830..86f84abb7fb6 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java @@ -571,7 +571,7 @@ public class PyFunctionImpl extends PyPresentableElementImpl imp @Override public PyAnnotation getAnnotation() { - return findChildByClass(PyAnnotation.class); + return getStubOrPsiChild(PyElementTypes.ANNOTATION); } @NotNull From 532ae5f155725f6bdcb2b2ba32faab64889d0b0a Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 21 Aug 2014 14:50:45 +0400 Subject: [PATCH 16/22] Tests of type unification for generic classes --- .../com/jetbrains/python/PyTypeTest.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/testSrc/com/jetbrains/python/PyTypeTest.java b/python/testSrc/com/jetbrains/python/PyTypeTest.java index b0e41bcffddc..b6ed1090edad 100644 --- a/python/testSrc/com/jetbrains/python/PyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypeTest.java @@ -835,6 +835,37 @@ public class PyTypeTest extends PyTestCase { "expr = (1,) + (True, 'spam') + ()"); } + public void testConstructorUnification() { + doTest("C[int]", + "class C(object):\n" + + " def __init__(self, x):\n" + + " '''\n" + + " :type x: T\n" + + " :rtype: C[T]\n" + + " '''\n" + + " pass\n" + + "\n" + + "expr = C(10)\n"); + } + + public void testGenericClassMethodUnification() { + doTest("int", + "class C(object):\n" + + " def __init__(self, x):\n" + + " '''\n" + + " :type x: T\n" + + " :rtype: C[T]\n" + + " '''\n" + + " pass\n" + + " def foo(self):\n" + + " '''\n" + + " :rtype: T\n" + + " '''\n" + + " pass\n" + + "\n" + + "expr = C(10).foo()\n"); + } + private static TypeEvalContext getTypeEvalContext(@NotNull PyExpression element) { return TypeEvalContext.userInitiated(element.getContainingFile()).withTracing(); } From 74df07fa7a6ec46366412d1e7a015e79fa4c4166 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 21 Aug 2014 18:10:57 +0400 Subject: [PATCH 17/22] Added heuristic: transform subscription expressions into operands in superclasses list Subscription expressions in the superclasses list are most likely to be used for parameterized classes, so we can just use the classes themselves for resolving superclasses and creating stubs. It is needed for parameterized classes in mypy. --- python/src/com/jetbrains/python/psi/impl/PyClassImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java b/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java index fad945b9ef3d..bb2104fc0c77 100644 --- a/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java +++ b/python/src/com/jetbrains/python/psi/impl/PyClassImpl.java @@ -186,6 +186,11 @@ public class PyClassImpl extends PyPresentableElementImpl implement } } } + // Heuristic: unfold Foo[Bar] to Foo for subscription expressions for superclasses + else if (expression instanceof PySubscriptionExpression) { + final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; + return subscriptionExpr.getOperand(); + } return expression; } From e34308dbbc7d98c73a9966b60e68c16329754aa9 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 21 Aug 2014 18:11:57 +0400 Subject: [PATCH 18/22] Support for parameterized classes --- .../codeInsight/PyTypingTypeProvider.java | 68 ++++++++++++++++++- .../com/jetbrains/python/PyTypingTest.java | 44 ++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index 2e7c3f3bece3..7ab97e6e5eb9 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -16,6 +16,7 @@ package com.jetbrains.python.codeInsight; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.util.QualifiedName; @@ -28,6 +29,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -41,6 +43,12 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { .put("typing.Tuple", PyNames.TUPLE) .build(); + private static ImmutableSet GENERIC_CLASSES = ImmutableSet.builder() + .add("typing.Generic") + .add("typing.AbstractGeneric") + .add("typing.Protocol") + .build(); + public PyType getParameterType(@NotNull PyNamedParameter param, @NotNull PyFunction func, @NotNull TypeEvalContext context) { final PyAnnotation annotation = param.getAnnotation(); if (annotation != null) { @@ -66,10 +74,68 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { return getTypingType(value, context); } } + final PyType constructorType = getGenericConstructorType(function, context); + if (constructorType != null) { + return constructorType; + } } return null; } + @Nullable + private static PyType getGenericConstructorType(@NotNull PyFunction function, @NotNull TypeEvalContext context) { + if (PyUtil.isInit(function)) { + final PyClass cls = function.getContainingClass(); + if (cls != null) { + final List genericTypes = collectGenericTypes(cls, context); + + final PyType elementType; + if (genericTypes.size() == 1) { + elementType = genericTypes.get(0); + } + else if (genericTypes.size() > 1) { + elementType = PyTupleType.create(cls, genericTypes.toArray(new PyType[genericTypes.size()])); + } + else { + elementType = null; + } + + if (elementType != null) { + return new PyCollectionTypeImpl(cls, false, elementType); + } + } + } + return null; + } + + @NotNull + private static List collectGenericTypes(@NotNull PyClass cls, @NotNull TypeEvalContext context) { + boolean isGeneric = false; + for (PyClass ancestor : cls.getAncestorClasses(context)) { + if (GENERIC_CLASSES.contains(ancestor.getQualifiedName())) { + isGeneric = true; + break; + } + } + if (isGeneric) { + final ArrayList results = new ArrayList(); + // XXX: Requires switching from stub to AST + for (PyExpression expr : cls.getSuperClassExpressions()) { + if (expr instanceof PySubscriptionExpression) { + final PyExpression indexExpr = ((PySubscriptionExpression)expr).getIndexExpression(); + if (indexExpr != null) { + final PyGenericType genericType = getGenericType(indexExpr, context); + if (genericType != null) { + results.add(genericType); + } + } + } + } + return results; + } + return Collections.emptyList(); + } + @Nullable private static PyType getTypingType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { final PyType unionType = getUnionType(expression, context); @@ -105,7 +171,7 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { } @Nullable - private static PyType getGenericType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + private static PyGenericType getGenericType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { final PsiElement resolved = resolve(expression, context); if (resolved instanceof PyTargetExpression) { final PyTargetExpression targetExpr = (PyTargetExpression)resolved; diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index d2841569041b..d592833a0d8a 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -159,6 +159,50 @@ public class PyTypingTest extends PyTestCase { " pass\n"); } + public void testParameterizedClass() { + doTest("C[int]", + "from typing import Generic, typevar\n" + + "\n" + + "T = typevar('T')\n" + + "\n" + + "class C(Generic[T]):\n" + + " def __init__(self, x: T):\n" + + " pass\n" + + "\n" + + "expr = C(10)\n"); + } + + public void testParameterizedClassMethod() { + doTest("int", + "from typing import Generic, typevar\n" + + "\n" + + "T = typevar('T')\n" + + "\n" + + "class C(Generic[T]):\n" + + " def __init__(self, x: T):\n" + + " pass\n" + + " def foo(self) -> T:\n" + + " pass\n" + + "\n" + + "expr = C(10).foo()\n"); + } + + public void testParameterizedClassInheritance() { + doTest("int", + "from typing import Generic, typevar\n" + + "\n" + + "T = typevar('T')\n" + + "\n" + + "class B(Generic[T]):\n" + + " def foo(self) -> T:\n" + + " pass\n" + + "class C(B[T]):\n" + + " def __init__(self, x: T):\n" + + " pass\n" + + "\n" + + "expr = C(10).foo()\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From 085a42d32d4095e113a7f8ef84bba8a69495b43a Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 21 Aug 2014 18:18:17 +0400 Subject: [PATCH 19/22] Tests for typing.AnyStr --- .../com/jetbrains/python/PyTypingTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index d592833a0d8a..5fc552e0d975 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -203,6 +203,27 @@ public class PyTypingTest extends PyTestCase { "expr = C(10).foo()\n"); } + public void testAnyStrUnification() { + doTest("bytes", + "from typing import AnyStr\n" + + "\n" + + "def foo(x: AnyStr) -> AnyStr:\n" + + " pass\n" + + "\n" + + "expr = foo(b'bar')\n"); + } + + public void testAnyStrForUnknown() { + doTest("str | bytes", + "from typing import AnyStr\n" + + "\n" + + "def foo(x: AnyStr) -> AnyStr:\n" + + " pass\n" + + "\n" + + "def bar(x):\n" + + " expr = foo(x)\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From 61c5615272b1335b89d3d400c7de371eac7d2f04 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Thu, 21 Aug 2014 18:39:03 +0400 Subject: [PATCH 20/22] Support for function types --- .../codeInsight/PyTypingTypeProvider.java | 34 +++++++++++++++++++ .../com/jetbrains/python/PyTypingTest.java | 8 +++++ 2 files changed, 42 insertions(+) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index 7ab97e6e5eb9..38b794fd2d25 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -154,6 +154,40 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { if (genericType != null) { return genericType; } + final PyType functionType = getFunctionType(expression, context); + if (functionType != null) { + return functionType; + } + return null; + } + + @Nullable + private static PyType getFunctionType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + if (expression instanceof PySubscriptionExpression) { + final PySubscriptionExpression subscriptionExpr = (PySubscriptionExpression)expression; + final PyExpression operand = subscriptionExpr.getOperand(); + final String operandName = resolveToQualifiedName(operand, context); + if ("typing.Function".equals(operandName)) { + final PyExpression indexExpr = subscriptionExpr.getIndexExpression(); + if (indexExpr instanceof PyTupleExpression) { + final PyTupleExpression tupleExpr = (PyTupleExpression)indexExpr; + final PyExpression[] elements = tupleExpr.getElements(); + if (elements.length == 2) { + final PyExpression parametersExpr = elements[0]; + if (parametersExpr instanceof PyListLiteralExpression) { + final List parameters = new ArrayList(); + final PyListLiteralExpression listExpr = (PyListLiteralExpression)parametersExpr; + for (PyExpression argExpr : listExpr.getElements()) { + parameters.add(new PyCallableParameterImpl(null, getType(argExpr, context))); + } + final PyExpression returnTypeExpr = elements[1]; + final PyType returnType = getType(returnTypeExpr, context); + return new PyCallableTypeImpl(parameters, returnType); + } + } + } + } + } return null; } diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index 5fc552e0d975..47d848c73dd7 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -224,6 +224,14 @@ public class PyTypingTest extends PyTestCase { " expr = foo(x)\n"); } + public void testFunctionType() { + doTest("(int, str) -> str", + "from typing import Function\n" + + "\n" + + "def foo(expr: Function[[int, str], str]):\n" + + " pass\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From 5a2ab2b55d5772bc6ff9894a5ee64baf7dc367f0 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Fri, 22 Aug 2014 15:14:08 +0400 Subject: [PATCH 21/22] Support for string-based 'typing' type annotations --- .../codeInsight/PyTypingTypeProvider.java | 23 +++++++++++++++++++ .../python/psi/resolve/PyResolveUtil.java | 8 ++++++- .../com/jetbrains/python/PyTypingTest.java | 17 ++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java index 38b794fd2d25..f03befac6c14 100644 --- a/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java +++ b/python/src/com/jetbrains/python/codeInsight/PyTypingTypeProvider.java @@ -17,11 +17,13 @@ package com.jetbrains.python.codeInsight; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.util.QualifiedName; import com.jetbrains.python.PyNames; import com.jetbrains.python.psi.*; +import com.jetbrains.python.psi.impl.PyExpressionCodeFragmentImpl; import com.jetbrains.python.psi.impl.PyPsiUtils; import com.jetbrains.python.psi.resolve.PyResolveContext; import com.jetbrains.python.psi.types.*; @@ -158,6 +160,27 @@ public class PyTypingTypeProvider extends PyTypeProviderBase { if (functionType != null) { return functionType; } + final PyType stringBasedType = getStringBasedType(expression, context); + if (stringBasedType != null) { + return stringBasedType; + } + return null; + } + + @Nullable + private static PyType getStringBasedType(@NotNull PyExpression expression, @NotNull TypeEvalContext context) { + if (expression instanceof PyStringLiteralExpression) { + // XXX: Requires switching from stub to AST + final String contents = ((PyStringLiteralExpression)expression).getStringValue(); + final Project project = expression.getProject(); + final PyExpressionCodeFragmentImpl codeFragment = new PyExpressionCodeFragmentImpl(project, "dummy.py", contents, false); + codeFragment.setContext(expression.getContainingFile()); + final PsiElement element = codeFragment.getFirstChild(); + if (element instanceof PyExpressionStatement) { + final PyExpression dummyExpr = ((PyExpressionStatement)element).getExpression(); + return getType(dummyExpr, context); + } + } return null; } diff --git a/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java b/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java index 838a4dbcc75a..f207169df77f 100644 --- a/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java +++ b/python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java @@ -101,7 +101,13 @@ public class PyResolveUtil { @Nullable PsiElement roof) { // Use real context here to enable correct completion and resolve in case of PyExpressionCodeFragment!!! final PsiElement realContext = PyPsiUtils.getRealContext(element); - final ScopeOwner originalOwner = ScopeUtil.getScopeOwner(realContext); + final ScopeOwner originalOwner; + if (realContext != element && realContext instanceof PyFile) { + originalOwner = (PyFile)realContext; + } + else { + originalOwner = ScopeUtil.getScopeOwner(realContext); + } final PsiElement parent = element.getParent(); final boolean isGlobalOrNonlocal = parent instanceof PyGlobalStatement || parent instanceof PyNonlocalStatement; ScopeOwner owner = originalOwner; diff --git a/python/testSrc/com/jetbrains/python/PyTypingTest.java b/python/testSrc/com/jetbrains/python/PyTypingTest.java index 47d848c73dd7..9513795cb5b7 100644 --- a/python/testSrc/com/jetbrains/python/PyTypingTest.java +++ b/python/testSrc/com/jetbrains/python/PyTypingTest.java @@ -232,6 +232,23 @@ public class PyTypingTest extends PyTestCase { " pass\n"); } + public void testTypeInStringLiteral() { + doTest("C", + "class C:\n" + + " def foo(self, expr: 'C'):\n" + + " pass\n"); + } + + public void testQualifiedTypeInStringLiteral() { + doTest("str", + "import typing\n" + + "\n" + + "def foo(x: 'typing.AnyStr') -> typing.AnyStr:\n" + + " pass\n" + + "\n" + + "expr = foo('bar')\n"); + } + private void doTest(@NotNull String expectedType, @NotNull String text) { myFixture.copyDirectoryToProject("typing", ""); myFixture.configureByText(PythonFileType.INSTANCE, text); From 79473fc932e0d6a1b6a5c0496505bd8113309fb3 Mon Sep 17 00:00:00 2001 From: Andrey Vlasovskikh Date: Fri, 22 Aug 2014 16:42:02 +0400 Subject: [PATCH 22/22] Removed unnecessary nullity checks --- .../python/psi/types/PyTypeChecker.java | 90 +++++++++---------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java b/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java index 9881d6cc106c..2e7c58c8201c 100644 --- a/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java +++ b/python/src/com/jetbrains/python/psi/types/PyTypeChecker.java @@ -428,41 +428,39 @@ public class PyTypeChecker { public static AnalyzeCallResults analyzeCall(@NotNull PyBinaryExpression expr, @NotNull TypeEvalContext context) { final PsiPolyVariantReference ref = expr.getReference(PyResolveContext.noImplicits().withTypeEvalContext(context)); final ResolveResult[] resolveResult; - if (ref != null) { - resolveResult = ref.multiResolve(false); - AnalyzeCallResults firstResults = null; - for (ResolveResult result : resolveResult) { - final PsiElement resolved = result.getElement(); - if (resolved instanceof PyTypedElement) { - final PyTypedElement typedElement = (PyTypedElement)resolved; - final PyType type = context.getType(typedElement); - if (!(type instanceof PyFunctionType)) { - return null; - } - final Callable callable = ((PyFunctionType)type).getCallable(); - final boolean isRight = PyNames.isRightOperatorName(typedElement.getName()); - final PyExpression arg = isRight ? expr.getLeftExpression() : expr.getRightExpression(); - final PyExpression receiver = isRight ? expr.getRightExpression() : expr.getLeftExpression(); - final PyParameter[] parameters = callable.getParameterList().getParameters(); - if (parameters.length >= 2) { - final PyNamedParameter param = parameters[1].getAsNamed(); - if (arg != null && param != null) { - final Map arguments = new LinkedHashMap(); - arguments.put(arg, param); - final AnalyzeCallResults results = new AnalyzeCallResults(callable, receiver, arguments); - if (firstResults == null) { - firstResults = results; - } - if (match(context.getType(param), context.getType(arg), context)) { - return results; - } + resolveResult = ref.multiResolve(false); + AnalyzeCallResults firstResults = null; + for (ResolveResult result : resolveResult) { + final PsiElement resolved = result.getElement(); + if (resolved instanceof PyTypedElement) { + final PyTypedElement typedElement = (PyTypedElement)resolved; + final PyType type = context.getType(typedElement); + if (!(type instanceof PyFunctionType)) { + return null; + } + final Callable callable = ((PyFunctionType)type).getCallable(); + final boolean isRight = PyNames.isRightOperatorName(typedElement.getName()); + final PyExpression arg = isRight ? expr.getLeftExpression() : expr.getRightExpression(); + final PyExpression receiver = isRight ? expr.getRightExpression() : expr.getLeftExpression(); + final PyParameter[] parameters = callable.getParameterList().getParameters(); + if (parameters.length >= 2) { + final PyNamedParameter param = parameters[1].getAsNamed(); + if (arg != null && param != null) { + final Map arguments = new LinkedHashMap(); + arguments.put(arg, param); + final AnalyzeCallResults results = new AnalyzeCallResults(callable, receiver, arguments); + if (firstResults == null) { + firstResults = results; + } + if (match(context.getType(param), context.getType(arg), context)) { + return results; } } } } - if (firstResults != null) { - return firstResults; - } + } + if (firstResults != null) { + return firstResults; } return null; } @@ -471,22 +469,20 @@ public class PyTypeChecker { public static AnalyzeCallResults analyzeCall(@NotNull PySubscriptionExpression expr, @NotNull TypeEvalContext context) { final PsiReference ref = expr.getReference(PyResolveContext.noImplicits().withTypeEvalContext(context)); final PsiElement resolved; - if (ref != null) { - resolved = ref.resolve(); - if (resolved instanceof PyTypedElement) { - final PyType type = context.getType((PyTypedElement)resolved); - if (type instanceof PyFunctionType) { - final Callable callable = ((PyFunctionType)type).getCallable(); - final PyParameter[] parameters = callable.getParameterList().getParameters(); - if (parameters.length == 2) { - final PyNamedParameter param = parameters[1].getAsNamed(); - if (param != null) { - final Map arguments = new LinkedHashMap(); - final PyExpression arg = expr.getIndexExpression(); - if (arg != null) { - arguments.put(arg, param); - return new AnalyzeCallResults(callable, expr.getOperand(), arguments); - } + resolved = ref.resolve(); + if (resolved instanceof PyTypedElement) { + final PyType type = context.getType((PyTypedElement)resolved); + if (type instanceof PyFunctionType) { + final Callable callable = ((PyFunctionType)type).getCallable(); + final PyParameter[] parameters = callable.getParameterList().getParameters(); + if (parameters.length == 2) { + final PyNamedParameter param = parameters[1].getAsNamed(); + if (param != null) { + final Map arguments = new LinkedHashMap(); + final PyExpression arg = expr.getIndexExpression(); + if (arg != null) { + arguments.put(arg, param); + return new AnalyzeCallResults(callable, expr.getOperand(), arguments); } } }