Merge remote-tracking branch 'origin/master'

python/helpers/generator3.py

@@ -24,7 +24,7 @@ but seemingly no one uses them in C extensions yet anyway.
 # * re.search-bound, ~30% time, in likes of builtins and _gtk with complex docstrings.
 # None of this can seemingly be easily helped. Maybe there's a simpler and faster parser library?
 
-VERSION = "1.114" # Must be a number-dot-number string, updated with each change that affects generated skeletons
+VERSION = "1.115" # Must be a number-dot-number string, updated with each change that affects generated skeletons
 # Note: DON'T FORGET TO UPDATE!
 
 VERSION_CONTROL_HEADER_FORMAT = '# from %s by generator %s'
@@ -911,6 +911,7 @@ class ModuleRedeclarator(object):
         ("_thread", None, "start_new"): ("(function, args, kwargs=None)", INT_LIT),
 
         ("itertools", "groupby", "__init__"): ("(self, iterable, key=None)", None),
+        ("itertools", None, "groupby"): ("(iterable, key=None)", LIST_LIT),
 
         # NOTE: here we stand on shaky ground providing sigs for 3rd-party modules, though well-known
         ("numpy.core.multiarray", "ndarray", "__array__") : ("(self, dtype=None)", None),

python/helpers/required_gen_version

@@ -6,7 +6,7 @@
 
 (default) 1.92 # anything not explicitly marked
 
-(built-in) 1.114 # skeletons of all built-in modules are built together
+(built-in) 1.115 # skeletons of all built-in modules are built together
 # Note: modules like itertools, etc are "(built-in)" and are ignored if given separately
 
 _fileio 1.101

python/src/com/jetbrains/python/codeInsight/dataflow/scope/ScopeUtil.java

@@ -3,6 +3,8 @@ package com.jetbrains.python.codeInsight.dataflow.scope;
 import com.intellij.codeInsight.controlflow.ControlFlow;
 import com.intellij.codeInsight.controlflow.Instruction;
 import com.intellij.psi.PsiElement;
+import com.intellij.psi.StubBasedPsiElement;
+import com.intellij.psi.stubs.StubElement;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache;
 import com.jetbrains.python.codeInsight.controlflow.ReadWriteInstruction;
@@ -52,6 +54,19 @@ public class ScopeUtil {
 
   @Nullable
   public static ScopeOwner getScopeOwner(PsiElement element) {
+    if (element instanceof StubBasedPsiElement) {
+      final StubElement stub = ((StubBasedPsiElement)element).getStub();
+      if (stub != null) {
+        StubElement parentStub = stub.getParentStub();
+        while (parentStub != null) {
+          final PsiElement parent = parentStub.getPsi();
+          if (parent instanceof ScopeOwner) {
+            return (ScopeOwner)parent;
+          }
+          parentStub = parentStub.getParentStub();
+        }
+      }
+    }
     return PsiTreeUtil.getParentOfType(element, ScopeOwner.class);
   }
 

python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java

@@ -6,6 +6,7 @@ import com.intellij.codeInsight.dataflow.map.DFAMap;
 import com.intellij.codeInsight.dataflow.map.DFAMapEngine;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiNamedElement;
+import com.jetbrains.cython.psi.CythonIncludeStatement;
 import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache;
 import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
 import com.jetbrains.python.codeInsight.dataflow.PyReachingDefsDfaInstance;
@@ -181,9 +182,8 @@ public class ScopeImpl implements Scope {
         if (node instanceof PsiNamedElement) {
           namedElements.put(node.getName(), (PsiNamedElement)node);
         }
-        // TODO: NameDefiners should be used only for defining lazily evaluated names
-        if (node instanceof NameDefiner && !(node instanceof PsiNamedElement ||
-                                             node instanceof PyParameterList)) {
+        // TODO: Cython-specific code
+        if (node instanceof PyStarImportElement || node instanceof PyImportElement || node instanceof CythonIncludeStatement) {
           nameDefiners.add((NameDefiner)node);
         }
         if (node instanceof ScopeOwner) {

python/src/com/jetbrains/python/codeInsight/stdlib/PyStdlibModuleMembersProvider.java

@@ -1,7 +1,10 @@
 package com.jetbrains.python.codeInsight.stdlib;
 
+import com.intellij.openapi.util.SystemInfo;
+import com.intellij.psi.PsiElement;
 import com.jetbrains.python.codeInsight.PyDynamicMember;
 import com.jetbrains.python.psi.PyFile;
+import com.jetbrains.python.psi.impl.PyQualifiedName;
 import com.jetbrains.python.psi.resolve.ResolveImportUtil;
 import com.jetbrains.python.psi.types.PyModuleMembersProvider;
 
@@ -14,8 +17,17 @@ import java.util.Collections;
 public class PyStdlibModuleMembersProvider extends PyModuleMembersProvider {
   @Override
   protected Collection<PyDynamicMember> getMembersByQName(PyFile module, String qName, ResolveImportUtil.PointInImport point) {
-    if (qName.equals("os") && point == ResolveImportUtil.PointInImport.AS_MODULE) {
-      return Collections.singletonList(new PyDynamicMember("path"));
+    if (qName.equals("os")) {
+      if (point == ResolveImportUtil.PointInImport.AS_MODULE) {
+        return Collections.singletonList(new PyDynamicMember("path"));
+      }
+      else if (point == ResolveImportUtil.PointInImport.NONE && module != null) {
+        final String name = SystemInfo.isWindows ? "ntpath" : "posixpath";
+        final PsiElement resolved = ResolveImportUtil.resolveModuleInRoots(PyQualifiedName.fromDottedString(name), module);
+        if (resolved != null) {
+          return Collections.singletonList(new PyDynamicMember("path", resolved));
+        }
+      }
     }
     return Collections.emptyList();
   }

python/src/com/jetbrains/python/inspections/PyPackageRequirementsInspection.java

@@ -184,9 +184,11 @@ public class PyPackageRequirementsInspection extends PyInspection {
                 final PsiElement element = reference.resolve();
                 if (element != null) {
                   final PsiFile file = element.getContainingFile();
-                  final VirtualFile virtualFile = file.getVirtualFile();
-                  if (ModuleUtil.moduleContainsFile(module, virtualFile, false)) {
-                    return;
+                  if (file != null) {
+                    final VirtualFile virtualFile = file.getVirtualFile();
+                    if (ModuleUtil.moduleContainsFile(module, virtualFile, false)) {
+                      return;
+                    }
                   }
                 }
               }

python/src/com/jetbrains/python/inspections/PyUnboundLocalVariableInspection.java

@@ -110,8 +110,10 @@ public class PyUnboundLocalVariableInspection extends PyInspection {
           }
         }
         if (owner instanceof PyFile) {
-          // Ignore builtins and variables that are not named elements, i. e. are defined by name definers
-          if (isBuiltin || scope.getNamedElement(name) == null) {
+          if (isBuiltin) {
+            return;
+          }
+          if (resolved != null && !PyUtil.inSameFile(node, resolved)) {
             return;
           }
           registerProblem(node, PyBundle.message("INSP.unbound.name.not.defined", name));

python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java

@@ -508,9 +508,8 @@ public class PyUnresolvedReferencesInspection extends PyInspection {
       registerProblem(point, description, hl_type, null, range, actions.toArray(new LocalQuickFix[actions.size()]));
     }
 
-    private static boolean ignoreUnresolvedMemberForType(PyType qtype, PsiReference reference, String refText) {
-      if (qtype instanceof PyNoneType || qtype instanceof PyTypeReference ||
-          (qtype instanceof PyUnionType && ((PyUnionType)qtype).isWeak())) {
+    private static boolean ignoreUnresolvedMemberForType(@NotNull PyType qtype, PsiReference reference, String refText) {
+      if (qtype instanceof PyNoneType || PyTypeChecker.isUnknown(qtype)) {
         // this almost always means that we don't know the type, so don't show an error in this case
         return true;
       }
@@ -715,7 +714,19 @@ public class PyUnresolvedReferencesInspection extends PyInspection {
 
       Set<PyImportStatementBase> unusedStatements = new HashSet<PyImportStatementBase>();
       final PyUnresolvedReferencesInspection suppressableInspection = new PyUnresolvedReferencesInspection();
+      PyQualifiedName packageQName = null;
+      List<String> dunderAll = null;
+
       for (NameDefiner unusedImport : unusedImports) {
+        if (packageQName == null) {
+          final PsiFile file = unusedImport.getContainingFile();
+          if (file instanceof PyFile) {
+            dunderAll = ((PyFile)file).getDunderAll();
+          }
+          if (file != null && PyUtil.isPackage(file)) {
+            packageQName = ResolveImportUtil.findShortestImportableQName(file);
+          }
+        }
         PyImportStatementBase importStatement = PsiTreeUtil.getParentOfType(unusedImport, PyImportStatementBase.class);
         if (importStatement != null && !unusedStatements.contains(importStatement) && !myUsedImports.contains(importStatement)) {
           if (suppressableInspection.isSuppressedFor(importStatement)) {
@@ -737,14 +748,28 @@ public class PyUnresolvedReferencesInspection extends PyInspection {
               continue;
             }
           }
+          PsiFileSystemItem importedElement;
           if (unusedImport instanceof PyImportElement) {
-            if (ResolveImportUtil.resolveImportElement((PyImportElement)unusedImport) == null) {
+            final PyImportElement importElement = (PyImportElement)unusedImport;
+            final PsiElement element = ResolveImportUtil.resolveImportElement(importElement);
+            if (element == null) {
               continue;
             }
+            if (dunderAll != null && dunderAll.contains(importElement.getVisibleName())) {
+              continue;
+            }
+            importedElement = element.getContainingFile();
           }
           else {
             assert importStatement instanceof PyFromImportStatement;
-            if (((PyFromImportStatement)importStatement).resolveImportSource() == null) {
+            importedElement = ((PyFromImportStatement)importStatement).resolveImportSource();
+            if (importedElement == null) {
+              continue;
+            }
+          }
+          if (packageQName != null && importedElement instanceof PsiFileSystemItem) {
+            final PyQualifiedName importedQName = ResolveImportUtil.findShortestImportableQName(importedElement);
+            if (importedQName != null && importedQName.matchesPrefix(packageQName)) {
               continue;
             }
           }

python/src/com/jetbrains/python/psi/impl/PyClassImpl.java

@@ -998,7 +998,7 @@ public class PyClassImpl extends PyPresentableElementImpl<PyClassStub> implement
       }
     }
     else {
-      PyResolveUtil.scopeCrawlUp(processor, this, this);
+      PyResolveUtil.scopeCrawlUp(processor, this, null, this);
     }
     return true;
   }

python/src/com/jetbrains/python/psi/impl/references/PyQualifiedReference.java

@@ -23,7 +23,6 @@ import com.intellij.util.indexing.FileBasedIndex;
 import com.jetbrains.cython.types.CythonStructType;
 import com.jetbrains.python.PyNames;
 import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
-import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil;
 import com.jetbrains.python.psi.*;
 import com.jetbrains.python.psi.impl.PyBuiltinCache;
 import com.jetbrains.python.psi.impl.PyImportedModule;
@@ -85,7 +84,7 @@ public class PyQualifiedReference extends PyReferenceImpl {
       }
     }
 
-    if ((qualifierType == null || qualifierType instanceof PyTypeReference) &&
+    if (PyTypeChecker.isUnknown(qualifierType) &&
         myContext.allowImplicits() && canQualifyAnImplicitName(qualifier, qualifierType)) {
       addImplicitResolveResults(referencedName, ret);
     }
@@ -363,9 +362,7 @@ public class PyQualifiedReference extends PyReferenceImpl {
     PyQualifiedName qualifierPath = PyQualifiedName.fromReferenceChain(PyResolveUtil.unwindQualifiers(qualifier));
     if (qualifierPath != null) {
       AssignmentCollectProcessor proc = new AssignmentCollectProcessor(qualifierPath);
-      final ScopeOwner roof = ScopeUtil.getResolveScopeOwner(qualifier);
-      PyResolveUtil.scopeCrawlUp(proc, qualifier, null, roof);
-      //PyResolveUtil.treeCrawlUp(proc, qualifier);
+      PyResolveUtil.treeCrawlUp(proc, qualifier);
       return proc.getResult();
     }
     else {

python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java

@@ -221,7 +221,15 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
             if (!latest.isEmpty()) {
               return latest;
             }
-            if (!isCythonLevel(myElement)) {
+            if (owner instanceof PyClass) {
+              final ScopeOwner classOwner = ScopeUtil.getScopeOwner(owner);
+              if (classOwner != null) {
+                final ResolveProcessor outerProcessor = new ResolveProcessor(referencedName);
+                PyResolveUtil.scopeCrawlUp(outerProcessor, classOwner, referencedName, roof);
+                uexpr = outerProcessor.getResult();
+              }
+            }
+            else if (!isCythonLevel(myElement)) {
               uexpr = null;
             }
           }
@@ -511,7 +519,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
     final CompletionVariantsProcessor processor = new CompletionVariantsProcessor(element);
     final ScopeOwner owner = realContext instanceof ScopeOwner ? (ScopeOwner)realContext : ScopeUtil.getScopeOwner(realContext);
     if (owner != null) {
-      PyResolveUtil.scopeCrawlUp(processor, owner, null);
+      PyResolveUtil.scopeCrawlUp(processor, owner, null, null);
     }
 
     // in a call, include function's arg names
@@ -520,7 +528,7 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
     // include builtin names
     final PyFile builtinsFile = PyBuiltinCache.getInstance(element).getBuiltinsFile();
     if (builtinsFile != null) {
-      PyResolveUtil.scopeCrawlUp(processor, builtinsFile, null);
+      PyResolveUtil.scopeCrawlUp(processor, builtinsFile, null, null);
     }
 
     if (underscores >= 2) {

python/src/com/jetbrains/python/psi/resolve/PyResolveUtil.java

@@ -103,8 +103,9 @@ public class PyResolveUtil {
     scopeCrawlUp(processor, owner, originalOwner, name, roof);
   }
 
-  public static void scopeCrawlUp(@NotNull PsiScopeProcessor processor, @NotNull ScopeOwner scopeOwner, @Nullable PsiElement roof) {
-    scopeCrawlUp(processor, scopeOwner, scopeOwner, null, roof);
+  public static void scopeCrawlUp(@NotNull PsiScopeProcessor processor, @NotNull ScopeOwner scopeOwner, @Nullable String name,
+                                  @Nullable PsiElement roof) {
+    scopeCrawlUp(processor, scopeOwner, scopeOwner, name, roof);
   }
 
   private static void scopeCrawlUp(@NotNull PsiScopeProcessor processor, @Nullable ScopeOwner scopeOwner,
@@ -112,26 +113,32 @@ public class PyResolveUtil {
     while (scopeOwner != null) {
       if (!(scopeOwner instanceof PyClass) || scopeOwner == originalScopeOwner) {
         final Scope scope = ControlFlowCache.getScope(scopeOwner);
+        boolean found = false;
         if (name != null) {
           final PsiElement resolved = scope.getNamedElement(name);
           if (resolved != null) {
             if (!processor.execute(resolved, ResolveState.initial())) {
-              return;
+              found = true;
             }
           }
         }
         else {
           for (PsiNamedElement element : scope.getNamedElements()) {
             if (!processor.execute(element, ResolveState.initial())) {
-              return;
+              found = true;
+              break;
             }
           }
         }
         for (NameDefiner definer : scope.getNameDefiners()) {
           if (!processor.execute(definer, ResolveState.initial())) {
-            return;
+            found = true;
+            break;
           }
         }
+        if (found) {
+          return;
+        }
       }
       if (scopeOwner == roof) {
         return;

python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java

@@ -565,7 +565,7 @@ public class ResolveImportUtil {
         components.set(0, "datetime");
         return PyQualifiedName.fromComponents(components);
       }
-      else if (head.equals("ntpath")) {
+      else if (head.equals("ntpath") | head.equals("posixpath")) {
         final List<String> result = new ArrayList<String>();
         result.add("os");
         components.set(0, "path");

python/src/com/jetbrains/python/psi/resolve/ResolveProcessor.java

@@ -17,6 +17,8 @@ import org.jetbrains.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.List;
 
+import static com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil.getScopeOwner;
+
 public class ResolveProcessor implements PsiScopeProcessor {
   @NotNull private final String myName;
   private PsiElement myResult = null;
@@ -147,13 +149,10 @@ public class ResolveProcessor implements PsiScopeProcessor {
   }
 
   private boolean setResult(PsiElement result, @Nullable PsiElement definer) {
-    if (myResult == null || getScope(myResult) == getScope(result) || (definer != null && getScope(myResult) == getScope(definer))) {
+    if (myResult == null || getScopeOwner(myResult) == getScopeOwner(result) ||
+        (definer != null && getScopeOwner(myResult) == getScopeOwner(definer))) {
       myResult = result;
     }
     return false;
   }
-
-  private static PsiElement getScope(PsiElement result) {
-    return PsiTreeUtil.getParentOfType(result, PyFunction.class, PyClass.class, PyFile.class);
-  }
 }

python/src/com/jetbrains/python/psi/types/PyModuleType.java

@@ -186,7 +186,7 @@ public class PyModuleType implements PyType { // Modules don't descend from obje
       @Override
       public void handleEvent(Event event, @Nullable Object associated) {
       }
-    }, owner, null);
+    }, owner, null, null);
     return visibleImports;
   }
 

python/testData/inspections/PyUnresolvedReferencesInspection/ReturnedQualifiedReferenceUnionType/a.py

@@ -0,0 +1,8 @@
+from b import f, g
+
+def test(x):
+    if x > 0:
+        out = f()
+    else:
+        out = g()
+    out.startswith('foo') # pass

python/testData/inspections/PyUnresolvedReferencesInspection/ReturnedQualifiedReferenceUnionType/b.py

@@ -0,0 +1,5 @@
+def f():
+    return 1
+
+def g():
+    return 'foo'

python/testData/inspections/PyUnresolvedReferencesInspection/UnusedImportsInPackage/a.py

@@ -0,0 +1,5 @@
+def g(x):
+    return x
+
+def h(x):
+    return x

python/testData/inspections/PyUnresolvedReferencesInspection/UnusedImportsInPackage/p1/__init__.py

@@ -0,0 +1,7 @@
+from .m1 import f
+from p1.m1 import f
+from m1 import f
+from a import g
+<warning descr="Unused import statement">from a import h</warning>
+
+__all__ = ['f', 'g']

python/testData/inspections/PyUnresolvedReferencesInspection/UnusedImportsInPackage/p1/m1.py

@@ -0,0 +1,2 @@
+def f(x):
+    pass

python/testData/resolve/ClassRedefinedField.py

@@ -0,0 +1,7 @@
+class Foo(object):
+    pass
+
+class Bar(object):
+    Foo = Foo
+#          <ref>
+

python/testData/resolve/multiFile/fromImportStarReassignment/FromImportStarReassignment.py

@@ -0,0 +1,4 @@
+from m1 import *
+
+foo = foo
+#      <ref>

python/testData/resolve/multiFile/fromImportStarReassignment/m1.py

@@ -0,0 +1,2 @@
+def foo():
+    return 'foo'

python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java

@@ -318,9 +318,7 @@ public class PyMultiFileResolveTest extends PyResolveTestCase {
     assertResolvesTo(CythonVariable.class, "foo");
   }
 
-  public void _testCythonCdefClassForwardInclude() {
-    // TODO: Currently we resolve named elements first and then ask name definers, so the result is the forward decl instead of the
-    // includeded class
+  public void testCythonCdefClassForwardInclude() {
     final PyTargetExpression target = assertResolvesTo(PyTargetExpression.class, "bar");
     final PyExpression value = target.findAssignedValue();
     assertNotNull(value);
@@ -386,6 +384,11 @@ public class PyMultiFileResolveTest extends PyResolveTestCase {
     assertResolvesTo(PyFunction.class, "m1");
   }
 
+  // PY-7026
+  public void testFromImportStarReassignment() {
+    assertResolvesTo(PyFunction.class, "foo");
+  }
+
   private void prepareTestDirectory() {
     final String testName = getTestName(true);
     myFixture.copyDirectoryToProject(testName, "");

python/testSrc/com/jetbrains/python/PyResolveTest.java

@@ -467,4 +467,9 @@ public class PyResolveTest extends PyResolveTestCase {
   public void testLambdaParameterInDefaultValue() {
     assertResolvesTo(PyNamedParameter.class, "xx");
   }
+
+  // PY-6540
+  public void testClassRedefinedField() {
+    assertResolvesTo(PyClass.class, "Foo");
+  }
 }

python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java

@@ -131,6 +131,16 @@ public class PyUnresolvedReferencesInspectionTest extends PyTestCase {
     doMultiFileTest("a.py");
   }
 
+  // PY-7022
+  public void testReturnedQualifiedReferenceUnionType() {
+    doMultiFileTest("a.py");
+  }
+
+  // PY-2668
+  public void testUnusedImportsInPackage() {
+    doMultiFileTest("p1/__init__.py");
+  }
+
   private void doTest() {
     myFixture.configureByFile(TEST_DIRECTORY + getTestName(true) + ".py");
     myFixture.enableInspections(PyUnresolvedReferencesInspection.class);