Merge branch 'python3.4'

python/helpers/tools/stdlib_packages.txt

@@ -6,6 +6,7 @@ argparse
 array
 ast
 asynchat
+asyncio
 asyncore
 atexit
 audiodev
@@ -68,6 +69,8 @@ dummy_thread
 dummy_threading
 email
 encodings
+ensurepip
+enum
 errno
 exceptions
 fcntl
@@ -150,6 +153,7 @@ os
 os2emxpath
 ossaudiodev
 parser
+pathlib
 pdb
 pickle
 pickletools
@@ -194,6 +198,7 @@ robotparser
 runpy
 sched
 select
+selectors
 sets
 sgmllib
 sha
@@ -219,6 +224,7 @@ sre_parse
 ssl
 stat
 statcache
+statistics
 statvfs
 string
 StringIO
@@ -254,6 +260,7 @@ token
 tokenize
 trace
 traceback
+tracemalloc
 tty
 turtle
 turtledemo

python/psi-api/src/com/jetbrains/python/psi/PyClass.java

@@ -136,11 +136,13 @@ public interface PyClass extends PsiNameIdentifierOwner, PyStatement, NameDefine
   /**
    * Finds a property with the specified name in the class or one of its ancestors.
    *
+   *
    * @param name of the property
+   * @param inherited
    * @return descriptor of property accessors, or null if such property does not exist.
    */
   @Nullable
-  Property findProperty(@NotNull String name);
+  Property findProperty(@NotNull String name, boolean inherited);
 
   /**
    * Apply a processor to every method, looking at superclasses in method resolution order as needed.

python/psi-api/src/com/jetbrains/python/psi/impl/PyTypeProvider.java

@@ -30,7 +30,7 @@ public interface PyTypeProvider {
   ExtensionPointName<PyTypeProvider> EP_NAME = ExtensionPointName.create("Pythonid.typeProvider");
 
   @Nullable
-  PyType getReferenceExpressionType(PyReferenceExpression referenceExpression, TypeEvalContext context);
+  PyType getReferenceExpressionType(@NotNull PyReferenceExpression referenceExpression, @NotNull TypeEvalContext context);
 
   @Nullable
   PyType getReferenceType(@NotNull PsiElement referenceTarget, TypeEvalContext context, @Nullable PsiElement anchor);

python/psi-api/src/com/jetbrains/python/psi/types/PyOverridingAncestorsClassMembersProvider.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2000-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.psi.types;
+
+/**
+ * @author vlan
+ */
+public interface PyOverridingAncestorsClassMembersProvider extends PyClassMembersProvider {
+}

python/psi-api/src/com/jetbrains/python/psi/types/PyTypeProviderBase.java

@@ -80,7 +80,7 @@ public class PyTypeProviderBase implements PyTypeProvider {
   };
 
   @Override
-  public PyType getReferenceExpressionType(PyReferenceExpression referenceExpression, TypeEvalContext context) {
+  public PyType getReferenceExpressionType(@NotNull PyReferenceExpression referenceExpression, @NotNull TypeEvalContext context) {
     return null;
   }
 

python/src/com/jetbrains/python/codeInsight/stdlib/PyStdlibTypeProvider.java

@@ -37,7 +37,8 @@ import java.util.Set;
  * @author yole
  */
 public class PyStdlibTypeProvider extends PyTypeProviderBase {
-  private static final Set<String> OPEN_FUNCTIONS = ImmutableSet.of("__builtin__.open", "io.open", "os.fdopen");
+  private static final Set<String> OPEN_FUNCTIONS = ImmutableSet.of("__builtin__.open", "io.open", "os.fdopen",
+                                                                    "pathlib.Path.open");
   private static final String BINARY_FILE_TYPE = "io.FileIO[bytes]";
   private static final String TEXT_FILE_TYPE = "io.TextIOWrapper[unicode]";
 
@@ -134,7 +135,10 @@ public class PyStdlibTypeProvider extends PyTypeProviderBase {
     for (Map.Entry<PyExpression, PyNamedParameter> entry : arguments.entrySet()) {
       final PyNamedParameter parameter = entry.getValue();
       if ("mode".equals(parameter.getName())) {
-        final PyExpression argument = entry.getKey();
+        PyExpression argument = entry.getKey();
+        if (argument instanceof PyKeywordArgument) {
+          argument = ((PyKeywordArgument)argument).getValueExpression();
+        }
         if (argument instanceof PyStringLiteralExpression) {
           mode = ((PyStringLiteralExpression)argument).getStringValue();
           break;

python/src/com/jetbrains/python/codeInsight/userSkeletons/PyUserSkeletonsClassMembersProvider.java

@@ -22,6 +22,7 @@ import com.jetbrains.python.psi.PyFunction;
 import com.jetbrains.python.psi.PyTargetExpression;
 import com.jetbrains.python.psi.types.PyClassMembersProviderBase;
 import com.jetbrains.python.psi.types.PyClassType;
+import com.jetbrains.python.psi.types.PyOverridingAncestorsClassMembersProvider;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -33,7 +34,7 @@ import java.util.List;
 /**
  * @author vlan
  */
-public class PyUserSkeletonsClassMembersProvider extends PyClassMembersProviderBase {
+public class PyUserSkeletonsClassMembersProvider extends PyClassMembersProviderBase implements PyOverridingAncestorsClassMembersProvider {
   @NotNull
   @Override
   public Collection<PyDynamicMember> getMembers(@NotNull PyClassType classType, PsiElement location) {

python/src/com/jetbrains/python/documentation/PyDocumentationBuilder.java

@@ -95,7 +95,7 @@ class PyDocumentationBuilder {
             PyType type = context.getType(qual);
             if (type instanceof PyClassType) {
               cls = ((PyClassType)type).getPyClass();
-              Property property = cls.findProperty(elementName);
+              Property property = cls.findProperty(elementName, true);
               if (property != null) {
                 is_property = true;
                 final AccessDirection dir = AccessDirection.of((PyElement)outer);

python/src/com/jetbrains/python/inspections/PyAttributeOutsideInitInspection.java

@@ -100,7 +100,7 @@ public class PyAttributeOutsideInitInspection extends PyInspection {
       for (Map.Entry<String, PyTargetExpression> attribute : attributes.entrySet()) {
         String attributeName = attribute.getKey();
         if (attributeName == null) continue;
-        final Property property = containingClass.findProperty(attributeName);
+        final Property property = containingClass.findProperty(attributeName, true);
         if (!attributesInInit.containsKey(attributeName) && property == null) {
           registerProblem(attribute.getValue(), PyBundle.message("INSP.attribute.$0.outside.init", attributeName),
                           new PyMoveAttributeToInitQuickFix());

python/src/com/jetbrains/python/inspections/PyPropertyAccessInspection.java

@@ -86,7 +86,7 @@ public class PyPropertyAccessInspection extends PyInspection {
               property = myPropertyCache.get(key);
             }
             else {
-              property = cls.findProperty(name);
+              property = cls.findProperty(name, true);
             }
             myPropertyCache.put(key, property); // we store nulls, too, to know that a property does not exist
             if (property != null) {

python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java

@@ -695,7 +695,7 @@ public class PyUnresolvedReferencesInspection extends PyInspection {
         if (overridesGetAttr(cls, myTypeEvalContext)) {
           return true;
         }
-        if (cls.findProperty(name) != null) {
+        if (cls.findProperty(name, true) != null) {
           return true;
         }
         if (PyUtil.hasUnresolvedAncestors(cls, myTypeEvalContext)) {

python/src/com/jetbrains/python/psi/impl/PyClassImpl.java

@@ -585,7 +585,7 @@ public class PyClassImpl extends PyPresentableElementImpl<PyClassStub> implement
 
   @Nullable
   @Override
-  public Property findProperty(@NotNull final String name) {
+  public Property findProperty(@NotNull final String name, boolean inherited) {
     Property property = findLocalProperty(name);
     if (property != null) {
       return property;
@@ -593,10 +593,12 @@ public class PyClassImpl extends PyPresentableElementImpl<PyClassStub> implement
     if (findMethodByName(name, false) != null || findClassAttribute(name, false) != null) {
       return null;
     }
-    for (PyClass aClass : getAncestorClasses()) {
-      final Property ancestorProperty = ((PyClassImpl)aClass).findLocalProperty(name);
-      if (ancestorProperty != null) {
-        return ancestorProperty;
+    if (inherited) {
+      for (PyClass aClass : getAncestorClasses()) {
+        final Property ancestorProperty = ((PyClassImpl)aClass).findLocalProperty(name);
+        if (ancestorProperty != null) {
+          return ancestorProperty;
+        }
       }
     }
     return null;

python/src/com/jetbrains/python/psi/impl/PyFunctionImpl.java

@@ -174,29 +174,28 @@ public class PyFunctionImpl extends PyPresentableElementImpl<PyFunctionStub> imp
   @Nullable
   @Override
   public PyType getReturnType(@NotNull TypeEvalContext context, @Nullable PyQualifiedExpression callSite) {
-    final PyType type = getGenericReturnType(context, callSite);
-
+    PyType type = getGenericReturnType(context, callSite);
     if (callSite == null) {
       return type;
     }
     final PyTypeChecker.AnalyzeCallResults results = PyTypeChecker.analyzeCallSite(callSite, context);
-
     if (PyTypeChecker.hasGenerics(type, context)) {
       if (results != null) {
         final Map<PyGenericType, PyType> substitutions = PyTypeChecker.unifyGenericCall(this, results.getReceiver(), results.getArguments(),
                                                                                         context);
-        if (substitutions != null) {
-          return PyTypeChecker.substitute(type, substitutions, context);
-        }
+        type = substitutions != null ? PyTypeChecker.substitute(type, substitutions, context) : null;
       }
-      return null;
+      else {
+        type = null;
+      }
+    }
+    if (results != null) {
+      type = replaceSelf(type, results.getReceiver(), context);
     }
     if (results != null && isDynamicallyEvaluated(results.getArguments().values(), context)) {
       return PyUnionType.createWeakType(type);
     }
-    else {
-      return type;
-    }
+    return type;
   }
 
   @Nullable
@@ -205,16 +204,36 @@ public class PyFunctionImpl extends PyPresentableElementImpl<PyFunctionStub> imp
    */
   public PyType getReturnTypeWithoutCallSite(@NotNull TypeEvalContext context,
                                              @Nullable PyExpression receiver) {
-    final PyType type = getGenericReturnType(context, null);
+    PyType type = getGenericReturnType(context, null);
     if (PyTypeChecker.hasGenerics(type, context)) {
-      final Map<PyGenericType, PyType> substitutions =
-        PyTypeChecker.unifyGenericCall(this, receiver, Maps.<PyExpression, PyNamedParameter>newHashMap(), context);
+      final Map<PyGenericType, PyType> substitutions = PyTypeChecker.unifyGenericCall(this, receiver,
+                                                                                      Maps.<PyExpression, PyNamedParameter>newHashMap(),
+                                                                                      context);
       if (substitutions != null) {
-        return PyTypeChecker.substitute(type, substitutions, context);
+        type = PyTypeChecker.substitute(type, substitutions, context);
+      }
+      else {
+        type = null;
       }
-      return null;
     }
-    return type;
+    return replaceSelf(type, receiver, context);
+  }
+
+  @Nullable
+  private PyType replaceSelf(@Nullable PyType returnType, @Nullable PyExpression receiver, @NotNull TypeEvalContext context) {
+    if (receiver != null) {
+      // TODO: Currently we substitute only simple subclass types, but we could handle union and collection types as well
+      if (returnType instanceof PyClassType) {
+        final PyClassType returnClassType = (PyClassType)returnType;
+        if (returnClassType.getPyClass() == getContainingClass()) {
+          final PyType receiverType = context.getType(receiver);
+          if (receiverType instanceof PyClassType && PyTypeChecker.match(returnType, receiverType, context)) {
+            return returnClassType.isDefinition() ? receiverType : ((PyClassType)receiverType).toInstance();
+          }
+        }
+      }
+    }
+    return returnType;
   }
 
   private static boolean isDynamicallyEvaluated(@NotNull Collection<PyNamedParameter> parameters, @NotNull TypeEvalContext context) {

python/src/com/jetbrains/python/psi/impl/PyReferenceExpressionImpl.java

@@ -265,7 +265,7 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere
     if (qualifierType instanceof PyClassType) {
       final PyClassType classType = (PyClassType)qualifierType;
       PyClass pyClass = classType.getPyClass();
-      Property property = pyClass.findProperty(name);
+      Property property = pyClass.findProperty(name, true);
       if (property != null) {
         if (classType.isDefinition()) {
           return Ref.<PyType>create(PyBuiltinCache.getInstance(pyClass).getObjectType(PyNames.PROPERTY));
@@ -292,7 +292,7 @@ public class PyReferenceExpressionImpl extends PyElementImpl implements PyRefere
   }
 
   @Nullable
-  private PyType getTypeFromProviders(TypeEvalContext context) {
+  private PyType getTypeFromProviders(@NotNull TypeEvalContext context) {
     for (PyTypeProvider provider : Extensions.getExtensions(PyTypeProvider.EP_NAME)) {
       try {
         final PyType type = provider.getReferenceExpressionType(this, context);

python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java

@@ -241,8 +241,8 @@ public class PyTargetExpressionImpl extends PyPresentableElementImpl<PyTargetExp
       if (exprType instanceof PyClassType) {
         final PyClass cls = ((PyClassType)exprType).getPyClass();
         final PyFunction enter = cls.findMethodByName(PyNames.ENTER, true);
-        if (enter != null) {
-          final PyType enterType = enter.getReturnType(context, null);
+        if (enter instanceof PyFunctionImpl) {
+          final PyType enterType = ((PyFunctionImpl)enter).getReturnTypeWithoutCallSite(context, expression);
           if (enterType != null) {
             return enterType;
           }

python/src/com/jetbrains/python/psi/types/PyClassTypeImpl.java

@@ -23,6 +23,7 @@ import com.intellij.openapi.extensions.Extensions;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.Pair;
+import com.intellij.openapi.util.Ref;
 import com.intellij.openapi.util.UserDataHolderBase;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiInvalidElementAccessException;
@@ -156,22 +157,9 @@ public class PyClassTypeImpl extends UserDataHolderBase implements PyClassType {
     }
 
     if (resolveContext.allowProperties()) {
-      Property property = myClass.findProperty(name);
-      if (property != null) {
-        Maybe<Callable> accessor = property.getByDirection(direction);
-        if (accessor.isDefined()) {
-          Callable accessor_code = accessor.value();
-          ResolveResultList ret = new ResolveResultList();
-          if (accessor_code != null) ret.poke(accessor_code, RatedResolveResult.RATE_NORMAL);
-          PyTargetExpression site = property.getDefinitionSite();
-          if (site != null) ret.poke(site, RatedResolveResult.RATE_LOW);
-          if (ret.size() > 0) {
-            return ret;
-          }
-          else {
-            return null;
-          } // property is found, but the required accessor is explicitly absent
-        }
+      final Ref<ResolveResultList> resultRef = findProperty(name, direction, true);
+      if (resultRef != null) {
+        return resultRef.get();
       }
     }
 
@@ -198,6 +186,11 @@ public class PyClassTypeImpl extends UserDataHolderBase implements PyClassType {
       return ResolveResultList.to(classMember);
     }
 
+    classMember = resolveByOverridingAncestorsMembersProviders(this, name, location);
+    if (classMember != null) {
+      return ResolveResultList.to(classMember);
+    }
+
     if (inherited) {
       for (PyClassLikeType type : myClass.getAncestorTypes(context)) {
         if (type instanceof PyClassType) {
@@ -237,12 +230,10 @@ public class PyClassTypeImpl extends UserDataHolderBase implements PyClassType {
       for (PyClassLikeType type : myClass.getAncestorTypes(context)) {
         if (type instanceof PyClassType) {
           final PyClass pyClass = ((PyClassType)type).getPyClass();
-          if (pyClass != null) {
-            PsiElement superMember = resolveByMembersProviders(new PyClassTypeImpl(pyClass, isDefinition()), name, location);
+          PsiElement superMember = resolveByMembersProviders(new PyClassTypeImpl(pyClass, isDefinition()), name, location);
 
-            if (superMember != null) {
-              return ResolveResultList.to(superMember);
-            }
+          if (superMember != null) {
+            return ResolveResultList.to(superMember);
           }
         }
       }
@@ -251,6 +242,28 @@ public class PyClassTypeImpl extends UserDataHolderBase implements PyClassType {
     return Collections.emptyList();
   }
 
+  private Ref<ResolveResultList> findProperty(String name, AccessDirection direction, boolean inherited) {
+    Ref<ResolveResultList> resultRef = null;
+    Property property = myClass.findProperty(name, inherited);
+    if (property != null) {
+      Maybe<Callable> accessor = property.getByDirection(direction);
+      if (accessor.isDefined()) {
+        Callable accessor_code = accessor.value();
+        ResolveResultList ret = new ResolveResultList();
+        if (accessor_code != null) ret.poke(accessor_code, RatedResolveResult.RATE_NORMAL);
+        PyTargetExpression site = property.getDefinitionSite();
+        if (site != null) ret.poke(site, RatedResolveResult.RATE_LOW);
+        if (ret.size() > 0) {
+          resultRef = Ref.create(ret);
+        }
+        else {
+          resultRef = Ref.create();
+        } // property is found, but the required accessor is explicitly absent
+      }
+    }
+    return resultRef;
+  }
+
   @Nullable
   private PyClassType getMetaclassType() {
     final PyClass metaClass = PyUtil.getMetaClass(myClass);
@@ -329,6 +342,17 @@ public class PyClassTypeImpl extends UserDataHolderBase implements PyClassType {
     return null;
   }
 
+  @Nullable
+  private static PsiElement resolveByOverridingAncestorsMembersProviders(PyClassType type, String name, @Nullable PyExpression location) {
+    for (PyClassMembersProvider provider : Extensions.getExtensions(PyClassMembersProvider.EP_NAME)) {
+      if (provider instanceof PyOverridingAncestorsClassMembersProvider) {
+        final PsiElement resolveResult = provider.resolveMember(type, name, location);
+        if (resolveResult != null) return resolveResult;
+      }
+    }
+    return null;
+  }
+
   @Nullable
   private static PsiElement resolveInner(@NotNull PyClass cls,
                                          boolean isDefinition,

python/testData/inspections/PyUnresolvedReferencesInspection/contextManagerSubclass.py

@@ -0,0 +1,12 @@
+class C(object):
+    def __enter__(self):
+        return self
+
+
+class D(C):
+    def foo(self):
+        pass
+
+
+with D() as cm:
+    cm.foo()  # pass

python/testData/inspections/PyUnresolvedReferencesInspection/returnSelfInSuperClass.py

@@ -0,0 +1,13 @@
+class C(object):
+    def get_self(self):
+        return self
+
+
+class D(C):
+    def foo(self):
+        pass
+
+
+d = D()
+print(d.foo())
+print(d.get_self().foo())  # pass

python/testSrc/com/jetbrains/python/PyClassicPropertyTest.java

@@ -38,7 +38,7 @@ public class PyClassicPropertyTest extends PyTestCase {
   public void testV1() throws Exception {
     Property p;
     Maybe<Callable> accessor;
-    p = myClass.findProperty("v1");
+    p = myClass.findProperty("v1", true);
     assertNotNull(p);
     assertNull(p.getDoc());
     PyTargetExpression site = p.getDefinitionSite();
@@ -62,7 +62,7 @@ public class PyClassicPropertyTest extends PyTestCase {
   public void testV2() throws Exception {
     Property p;
     Maybe<Callable> accessor;
-    p = myClass.findProperty("v2");
+    p = myClass.findProperty("v2", true);
     assertNotNull(p);
     assertEquals("doc of v2", p.getDoc());
     PyTargetExpression site = p.getDefinitionSite();
@@ -86,7 +86,7 @@ public class PyClassicPropertyTest extends PyTestCase {
 
   public void testV3() throws Exception {
     Maybe<Callable> accessor;
-    Property p = myClass.findProperty("v3");
+    Property p = myClass.findProperty("v3", true);
     assertNotNull(p);
     assertNull(p.getDoc());
     PyTargetExpression site = p.getDefinitionSite();

python/testSrc/com/jetbrains/python/PyDecoratedPropertyTest.java

@@ -35,7 +35,7 @@ public class PyDecoratedPropertyTest extends PyTestCase {
     Property p;
     Maybe<Callable> accessor;
     final String name = "w1";
-    p = myClass.findProperty(name);
+    p = myClass.findProperty(name, true);
     assertNotNull(p);
     assertNull(p.getDoc());
     assertNull(p.getDefinitionSite());
@@ -60,7 +60,7 @@ public class PyDecoratedPropertyTest extends PyTestCase {
     Property p;
     Maybe<Callable> accessor;
     final String name = "w2";
-    p = myClass.findProperty(name);
+    p = myClass.findProperty(name, true);
     assertNotNull(p);
     assertNull(p.getDoc());
     assertNull(p.getDefinitionSite());

python/testSrc/com/jetbrains/python/PyStubsTest.java

@@ -118,12 +118,12 @@ public class PyStubsTest extends PyTestCase {
       pyClass = classes.get(1);
       assertEquals("BarClass", pyClass.getName());
 
-      Property prop = pyClass.findProperty("value");
+      Property prop = pyClass.findProperty("value", true);
       Maybe<Callable> maybe_function = prop.getGetter();
       assertTrue(maybe_function.isDefined());
       assertEquals(pyClass.getMethods()[0], maybe_function.value());
 
-      Property setvalueProp = pyClass.findProperty("setvalue");
+      Property setvalueProp = pyClass.findProperty("setvalue", true);
       Maybe<Callable> setter = setvalueProp.getSetter();
       assertTrue(setter.isDefined());
       assertEquals("__set", setter.value().getName());
@@ -131,7 +131,7 @@ public class PyStubsTest extends PyTestCase {
       // properties by decorator
       pyClass = classes.get(2);
       assertEquals("BazClass", pyClass.getName());
-      prop = pyClass.findProperty("x");
+      prop = pyClass.findProperty("x", true);
       maybe_function = prop.getGetter();
       assertTrue(maybe_function.isDefined());
       assertEquals(pyClass.getMethods()[0], maybe_function.value());

python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java

@@ -326,6 +326,16 @@ public class PyUnresolvedReferencesInspectionTest extends PyTestCase {
     doTest();
   }
 
+  // PY-10977
+  public void testContextManagerSubclass() {
+    doTest();
+  }
+
+  // PY-11413
+  public void testReturnSelfInSuperClass() {
+    doTest();
+  }
+
   private void doTest() {
     myFixture.configureByFile(TEST_DIRECTORY + getTestName(true) + ".py");
     myFixture.enableInspections(PyUnresolvedReferencesInspection.class);