Merge remote-tracking branch 'origin/master'

python/build/pycharm_build.gant

@@ -1,4 +1,4 @@
-  import org.jetbrains.jps.LayoutInfo
+import org.jetbrains.jps.LayoutInfo
 
 import static org.jetbrains.jps.idea.IdeaProjectLoader.guessHome
 
@@ -325,11 +325,11 @@ private layoutFull(Map args, String target, Set usedJars) {
         }
 
         jar("pycharm-pydev-src.zip") {
-          fileset(dir: "$home/python/pydevSrc")
+          fileset(dir: "$pythonCommunityHome/pydevSrc")
         }
         jar("pycharm-openapi-src.zip") {
-          fileset(dir: "$home/python/openapi/src")
-          fileset(dir: "$home/python/psi-api/src")
+          fileset(dir: "$pythonCommunityHome/openapi/src")
+          fileset(dir: "$pythonCommunityHome/python/psi-api/src")
         }
       }
     }
@@ -341,7 +341,7 @@ private layoutFull(Map args, String target, Set usedJars) {
     }
 
     dir("helpers") {
-      fileset(dir: "$home/python/helpers")
+      fileset(dir: "$pythonCommunityHome/helpers")
     }
 
     dir("license") {

python/build/pycharm_community_build.gant

@@ -288,11 +288,11 @@ private layoutFull(Map args, String target, Set usedJars) {
         }
 
         jar("pycharm-pydev-src.zip") {
-          fileset(dir: "$home/python/pydevSrc")
+          fileset(dir: "$pythonCommunityHome/pydevSrc")
         }
         jar("pycharm-openapi-src.zip") {
-          fileset(dir: "$home/python/openapi/src")
-          fileset(dir: "$home/python/psi-api/src")
+          fileset(dir: "$pythonCommunityHome/openapi/src")
+          fileset(dir: "$pythonCommunityHome/psi-api/src")
         }
       }
     }
@@ -304,7 +304,7 @@ private layoutFull(Map args, String target, Set usedJars) {
     }
 
     dir("helpers") {
-      fileset(dir: "$home/python/helpers")
+      fileset(dir: "$pythonCommunityHome/helpers")
     }
 
     dir("license") {

python/build/python_plugin_build.gant

@@ -30,7 +30,7 @@ setProperty("plugindir", "${zipdir}/python")
 setProperty("zipname", "python-${pluginRevision}.zip")
 
 setProperty("srcDir", "${pluginHome}/src")
-setProperty("restSrcDir", "${pluginHome}/rest/src")
+setProperty("restSrcDir", "${pluginHome}/community/rest/src")
 setProperty("localizationSrcDir", "${pluginHome}/localization/src")
 
 setProperty("resourcesDir", "${pluginHome}/resources")
@@ -274,9 +274,9 @@ target(name: "compile", description: "Compile module python") {
       include(name: "resources")
       include(name: "src")
       include(name: "pluginSrc")
-      include(name: "pydevSrc")
-      include(name: "openapi/src")
-      include(name: "psi-api/src")
+      include(name: "community/pydevSrc")
+      include(name: "community/openapi/src")
+      include(name: "community/psi-api/src")
       include(name: "community/src")
       include(name: "pluginResources")
     }
@@ -325,7 +325,7 @@ target(name: "compile", description: "Compile module python") {
       patternset(refid: "resources.pt")
       type(type: "file")
     }
-    fileset(dir: "${pluginHome}/psi-api/resources") {
+    fileset(dir: "${pluginHome}/community/psi-api/resources") {
       patternset(refid: "resources.pt")
       type(type: "file")
     }
@@ -386,7 +386,7 @@ target(name: "zip", description: "Generate zip plugin file") {
 
   ant.mkdir(dir: "${plugindir}/helpers")
   ant.copy(todir: "${plugindir}/helpers") {
-    fileset(dir: "${pluginHome}/helpers") {
+    fileset(dir: "${pluginHome}/community/helpers") {
       include(name: "**/*")
     }
   }

python/helpers/pycharm/pycharm_run_utils.py

@@ -32,6 +32,9 @@ def getModuleName(prefix, cnt):
   return prefix + "%" + str(cnt)
 
 def insert_to_sys_path(script_path):
+  print(script_path)
+  for s in sys.path:
+      print(s)
   while script_path in sys.path:
     sys.path.remove(script_path)
   sys.path.insert(0, script_path)

python/helpers/pydev/pydev_monkey.py

@@ -27,12 +27,17 @@ def patch_args(args):
             return args
 
         if is_python(args[0]):
-            if '-c' == args[1]:
+            try:
+                indC = args.index('-c')
+            except KeyError:
+                indC = -1
+
+            if indC != -1:
                 import pydevd
                 host, port = pydevd.dispatch()
 
                 if port is not None:
-                    args[2] = "import sys; sys.path.append('%s'); import pydevd; pydevd.settrace(host='%s', port=%s, suspend=False); %s"%(helpers, host, port, args[2])
+                    args[indC + 1] = "import sys; sys.path.append('%s'); import pydevd; pydevd.settrace(host='%s', port=%s, suspend=False); %s"%(helpers, host, port, args[indC + 1])
                     return args
             else:
                 new_args.append(args[0])

python/pluginSrc/python-community-plugin.iml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="lang-api" />
+    <orderEntry type="module" module-name="python-community" />
+    <orderEntry type="module" module-name="idea-ui" />
+    <orderEntry type="module" module-name="platform-impl" />
+    <orderEntry type="module" module-name="openapi" />
+    <orderEntry type="module" module-name="lang-impl" />
+  </component>
+</module>
+

python/python-helpers.iml

@@ -2,8 +2,8 @@
 <module type="JAVA_MODULE" version="4">
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
-    <content url="file://$MODULE_DIR$/helpers">
-      <sourceFolder url="file://$MODULE_DIR$/helpers" isTestSource="false" />
+    <content url="file://$MODULE_DIR$/community/helpers">
+      <sourceFolder url="file://$MODULE_DIR$/community/helpers" isTestSource="false" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

python/python-plugin.iml

@@ -16,6 +16,7 @@
     <orderEntry type="module" module-name="ultimate-verifier" />
     <orderEntry type="module" module-name="xml" />
     <orderEntry type="module" module-name="webide-api" />
+    <orderEntry type="module" module-name="python-community-plugin" />
   </component>
 </module>
 

python/python-pydev.iml

@@ -2,8 +2,8 @@
 <module type="JAVA_MODULE" version="4">
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
-    <content url="file://$MODULE_DIR$/pydevSrc">
-      <sourceFolder url="file://$MODULE_DIR$/pydevSrc" isTestSource="false" />
+    <content url="file://$MODULE_DIR$/community/pydevSrc">
+      <sourceFolder url="file://$MODULE_DIR$/community/pydevSrc" isTestSource="false" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

python/src/com/jetbrains/python/codeInsight/dataflow/scope/impl/ScopeImpl.java

@@ -226,19 +226,7 @@ public class ScopeImpl implements Scope {
       }
     });
 
-    Collections.sort(importedNameDefiners, new Comparator<NameDefiner>() {
-      @Override
-      public int compare(NameDefiner d1, NameDefiner d2) {
-        return getPriority(d2) - getPriority(d1);
-      }
-
-      private int getPriority(NameDefiner nameDefiner) {
-        if (nameDefiner instanceof PyStarImportElement) {
-          return -10;
-        }
-        return 0;
-      }
-    });
+    Collections.reverse(importedNameDefiners);
 
     myNamedElements = namedElements;
     myImportedNameDefiners = importedNameDefiners;

python/src/com/jetbrains/python/debugger/PyExceptionBreakpointType.java

@@ -33,7 +33,7 @@ public class PyExceptionBreakpointType
   private static final String BASE_EXCEPTION = "BaseException";
 
   public PyExceptionBreakpointType() {
-    super("python-exception", "Python Exception Breakpoint", false);
+    super("python-exception", "Python Exception Breakpoint");
   }
 
   @NotNull
@@ -73,6 +73,7 @@ public class PyExceptionBreakpointType
       final String qualifiedName = pyClass.getQualifiedName();
       assert qualifiedName != null : "Qualified name of the class shouldn't be null";
       return ApplicationManager.getApplication().runWriteAction(new Computable<XBreakpoint<PyExceptionBreakpointProperties>>() {
+        @Override
         public XBreakpoint<PyExceptionBreakpointProperties> compute() {
           return XDebuggerManager.getInstance(project).getBreakpointManager()
             .addBreakpoint(PyExceptionBreakpointType.this, new PyExceptionBreakpointProperties(qualifiedName));
@@ -85,6 +86,7 @@ public class PyExceptionBreakpointType
   private static class PyExceptionCachingFilter implements AbstractTreeClassChooserDialog.Filter<PyClass> {
     private final HashMap<Integer, Pair<WeakReference<PyClass>, Boolean>> processedElements = Maps.newHashMap();
 
+    @Override
     public boolean isAccepted(@NotNull final PyClass pyClass) {
       final VirtualFile virtualFile = pyClass.getContainingFile().getVirtualFile();
       if (virtualFile == null) {
@@ -194,6 +196,7 @@ public class PyExceptionBreakpointType
       panel.setBorder(IdeBorderFactory.createTitledBorder("Activation policy", true));
 
       ActionListener listener = new ActionListener() {
+        @Override
         public void actionPerformed(ActionEvent e) {
           setRadioButtonsEnabled();
         }

python/src/com/jetbrains/python/inspections/PyNoneFunctionAssignmentInspection.java

@@ -3,11 +3,10 @@ package com.jetbrains.python.inspections;
 import com.intellij.codeInspection.LocalInspectionToolSession;
 import com.intellij.codeInspection.ProblemsHolder;
 import com.intellij.psi.PsiElementVisitor;
+import com.intellij.util.containers.hash.HashMap;
 import com.jetbrains.python.PyBundle;
-import com.jetbrains.python.psi.Callable;
-import com.jetbrains.python.psi.PyAssignmentStatement;
-import com.jetbrains.python.psi.PyCallExpression;
-import com.jetbrains.python.psi.PyExpression;
+import com.jetbrains.python.psi.*;
+import com.jetbrains.python.psi.search.PyOverridingMethodsSearch;
 import com.jetbrains.python.psi.types.PyNoneType;
 import com.jetbrains.python.psi.types.PyType;
 import com.jetbrains.python.psi.types.PyTypeChecker;
@@ -16,6 +15,8 @@ import org.jetbrains.annotations.Nls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.Map;
+
 /**
  * User: ktisha
  *
@@ -41,6 +42,8 @@ public class PyNoneFunctionAssignmentInspection extends PyInspection {
 
 
   private static class Visitor extends PyInspectionVisitor {
+    private final Map<PyFunction, Boolean> myHasInheritors = new HashMap<PyFunction, Boolean>();
+
     public Visitor(@Nullable ProblemsHolder holder, @NotNull LocalInspectionToolSession session) {
       super(holder, session);
     }
@@ -56,11 +59,30 @@ public class PyNoneFunctionAssignmentInspection extends PyInspection {
           final PyTypeChecker.AnalyzeCallResults analyzeCallResults = PyTypeChecker.analyzeCall(((PyCallExpression)value), myTypeEvalContext);
           if (analyzeCallResults != null) {
             final Callable callable = analyzeCallResults.getCallable();
-            if (PySdkUtil.isElementInSkeletons(callable)) return;
+            if (PySdkUtil.isElementInSkeletons(callable)) {
+              return;
+            }
+            if (callable instanceof PyFunction) {
+              final PyFunction function = (PyFunction)callable;
+              // Currently we don't infer types returned by decorators
+              if (hasInheritors(function) || PyUtil.hasCustomDecorators(function)) {
+                return;
+              }
+            }
             registerProblem(node, PyBundle.message("INSP.none.function.assignment", callee.getName()));
           }
         }
       }
     }
+
+    private boolean hasInheritors(@NotNull PyFunction function) {
+      final Boolean cached = myHasInheritors.get(function);
+      if (cached != null) {
+        return cached;
+      }
+      final boolean result = PyOverridingMethodsSearch.search(function, true).findFirst() != null;
+      myHasInheritors.put(function, result);
+      return result;
+    }
   }
 }

python/src/com/jetbrains/python/inspections/PyRedeclarationInspection.java

@@ -6,6 +6,7 @@ import com.intellij.codeInspection.LocalInspectionToolSession;
 import com.intellij.codeInspection.LocalQuickFix;
 import com.intellij.codeInspection.ProblemHighlightType;
 import com.intellij.codeInspection.ProblemsHolder;
+import com.intellij.openapi.util.Ref;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiElementVisitor;
 import com.intellij.psi.PsiNameIdentifierOwner;
@@ -108,26 +109,23 @@ public class PyRedeclarationInspection extends PyInspection {
         if (startInstruction < 0) {
           return;
         }
+        final Ref<PsiElement> readElementRef = Ref.create(null);
+        final Ref<PsiElement> writeElementRef = Ref.create(null);
         ControlFlowUtil.iteratePrev(startInstruction, instructions, new Function<Instruction, ControlFlowUtil.Operation>() {
           @Override
           public ControlFlowUtil.Operation fun(Instruction instruction) {
             if (instruction instanceof ReadWriteInstruction && instruction.num() != startInstruction) {
               final ReadWriteInstruction rwInstruction = (ReadWriteInstruction)instruction;
               if (name.equals(rwInstruction.getName())) {
-                if (rwInstruction.getAccess().isWriteAccess()) {
-                  final List<LocalQuickFix> quickFixes = new ArrayList<LocalQuickFix>();
-                  final PsiElement originalElement = rwInstruction.getElement();
-                  if (originalElement != null && originalElement != element) {
-                    if (suggestRename(element, originalElement)) {
-                      quickFixes.add(new PyRenameElementQuickFix());
+                final PsiElement originalElement = rwInstruction.getElement();
+                if (originalElement != null) {
+                  if (rwInstruction.getAccess().isReadAccess()) {
+                    readElementRef.set(originalElement);
+                  }
+                  if (rwInstruction.getAccess().isWriteAccess()) {
+                    if (originalElement != element) {
+                      writeElementRef.set(originalElement);
                     }
-                    final PsiElement identifier = element.getNameIdentifier();
-                    registerProblem(identifier != null ? identifier : element,
-                                    PyBundle.message("INSP.redeclared.name", name),
-                                    ProblemHighlightType.GENERIC_ERROR_OR_WARNING,
-                                    null,
-                                    quickFixes.toArray(new LocalQuickFix[quickFixes.size()]));
-                    return ControlFlowUtil.Operation.BREAK;
                   }
                 }
                 return ControlFlowUtil.Operation.CONTINUE;
@@ -136,6 +134,19 @@ public class PyRedeclarationInspection extends PyInspection {
             return ControlFlowUtil.Operation.NEXT;
           }
         });
+        final PsiElement writeElement = writeElementRef.get();
+        if (writeElement != null && readElementRef.get() == null) {
+          final List<LocalQuickFix> quickFixes = new ArrayList<LocalQuickFix>();
+          if (suggestRename(element, writeElement)) {
+            quickFixes.add(new PyRenameElementQuickFix());
+          }
+          final PsiElement identifier = element.getNameIdentifier();
+          registerProblem(identifier != null ? identifier : element,
+                          PyBundle.message("INSP.redeclared.name", name),
+                          ProblemHighlightType.GENERIC_ERROR_OR_WARNING,
+                          null,
+                          quickFixes.toArray(new LocalQuickFix[quickFixes.size()]));
+        }
       }
     }
 

python/src/com/jetbrains/python/psi/impl/PyFileImpl.java

@@ -97,28 +97,11 @@ public class PyFileImpl extends PsiFileBase implements PyFile, PyExpression {
             addImportElementDeclaration(importElement, localDeclarations, ambiguousDeclarations);
           }
         }
-        if (PyNames.INIT_DOT_PY.equals(getName())) {
-          final PyQualifiedName qName = fromImportStatement.getImportSourceQName();
-          if (qName != null && !localDeclarations.containsKey(qName.toString())) {
-            localDeclarations.put(qName.getLastComponent(), fromImportStatement);
-          }
-        }
       }
       else if (child instanceof PyImportStatement) {
         final PyImportStatement importStatement = (PyImportStatement)child;
         for (PyImportElement importElement : importStatement.getImportElements()) {
           addImportElementDeclaration(importElement, localDeclarations, ambiguousDeclarations);
-          if (PyNames.INIT_DOT_PY.equals(getName())) {
-            final PyQualifiedName qName = importElement.getImportedQName();
-            final PsiDirectory containingDirectory = getContainingDirectory();
-            if (qName != null && qName.getComponentCount() > 1 && containingDirectory != null) {
-              String parentPackage = containingDirectory.getName();
-              final List<String> components = qName.getComponents();
-              if (components.get(components.size() - 2).equals(parentPackage)) {
-                localDeclarations.put(components.get(components.size()-1), importElement);
-              }
-            }
-          }
         }
       }
       else if (child instanceof NameDefiner) {

python/testData/inspections/PyNoneFunctionAssignmentInspection/decoratedMethod.py

@@ -0,0 +1,31 @@
+import abc
+from abc import abstractmethod
+
+
+def decorator(f):
+    return f
+
+
+class C(object):
+    __metaclass__ = abc.ABCMeta
+
+    @abstractmethod
+    def foo(self):
+        pass
+
+    @abc.abstractmethod
+    def bar(self):
+        pass
+
+    @decorator
+    def baz(self):
+        pass
+
+    def quux(self):
+        pass
+
+    def test(self):
+        a = self.foo()
+        b = self.bar()
+        c = self.baz()
+        <weak_warning descr="Function 'quux' doesn't return anything">d = self.quux()</weak_warning>

python/testData/inspections/PyNoneFunctionAssignmentInspection/methodWithInheritors.py

@@ -0,0 +1,15 @@
+class C(object):
+    def foo(self):
+        pass
+
+    def bar(self):
+        pass
+
+    def test(self):
+        x = self.foo()
+        <weak_warning descr="Function 'bar' doesn't return anything">y = self.bar()</weak_warning>
+
+
+class D(C):
+    def foo(self):
+        return 2

python/testData/inspections/PyRedeclarationInspection/test.py

@@ -87,3 +87,9 @@ class TestForLoopBody:
     for _ in [1, 2, 3]:
         def <warning descr="Redeclared 'foo' defined above without usage">foo</warning>():
             pass
+
+
+# PY-10839
+class TestNestedComprehension:
+    x = [[n for _ in []] for n in []]
+    n = 2

python/testData/inspections/unusedImport/usedLastImport/a.py

@@ -0,0 +1 @@
+foo = 0

python/testData/inspections/unusedImport/usedLastImport/b.py

@@ -0,0 +1 @@
+foo = 1

python/testData/inspections/unusedImport/usedLastImport/test1.py

@@ -0,0 +1,5 @@
+<warning descr="Unused import statement">from a import foo</warning>
+from b import foo
+
+
+print(foo)

python/testData/resolve/multiFile/fromPackageModuleImportElementNamedAsModule/FromPackageModuleImportElementNamedAsModule.py

@@ -0,0 +1,4 @@
+from p1 import foo
+
+foo()
+#<ref>

python/testData/resolve/multiFile/fromPackageModuleImportElementNamedAsModule/p1/__init__.py

@@ -0,0 +1 @@
+from p1.foo import foo

python/testData/resolve/multiFile/fromPackageModuleImportElementNamedAsModule/p1/foo.py

@@ -0,0 +1,2 @@
+def foo():
+    pass

python/testData/resolve/multiFile/fromPackageModuleImportStarElementNamedAsModule/FromPackageModuleImportStarElementNamedAsModule.py

@@ -0,0 +1,4 @@
+from p1 import foo
+
+foo()
+#<ref>

python/testData/resolve/multiFile/fromPackageModuleImportStarElementNamedAsModule/p1/__init__.py

@@ -0,0 +1 @@
+from p1.foo import *

python/testData/resolve/multiFile/fromPackageModuleImportStarElementNamedAsModule/p1/foo.py

@@ -0,0 +1,2 @@
+def foo():
+    pass

python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java

@@ -437,6 +437,19 @@ public class PyMultiFileResolveTest extends PyResolveTestCase {
     assertResolvesTo(PyTargetExpression.class, "foo");
   }
 
+  // PY-10819
+  public void testFromPackageModuleImportElementNamedAsModule() {
+    assertResolvesTo(PyFunction.class, "foo");
+    final PsiManager psiManager = myFixture.getPsiManager();
+    assertNotParsed((PyFile)psiManager.findFile(myFixture.findFileInTempDir("p1/__init__.py")));
+    assertNotParsed((PyFile)psiManager.findFile(myFixture.findFileInTempDir("p1/foo.py")));
+  }
+
+  // PY-10819
+  public void testFromPackageModuleImportStarElementNamedAsModule() {
+    assertResolvesTo(PyFunction.class, "foo");
+  }
+
   private void prepareTestDirectory() {
     final String testName = getTestName(true);
     myFixture.copyDirectoryToProject(testName, "");

python/testSrc/com/jetbrains/python/PyStubsTest.java

@@ -286,22 +286,6 @@ public class PyStubsTest extends PyTestCase {
     assertNotParsed(file);
   }
 
-  public void testInitPyExportsSubpackageImport() {
-    myFixture.copyDirectoryToProject("../resolve/multiFile/importPackageIntoSelf", "");
-    final PyFileImpl initPy = (PyFileImpl) myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("mygame/__init__.py"));
-    final PsiElement element = initPy.findExportedName("display");
-    assertTrue(element instanceof PyImportElement);
-    assertNotParsed(initPy);
-  }
-
-  public void testInitPyExportsSubpackageFromImport() {
-    myFixture.copyDirectoryToProject("../resolve/multiFile/fromImportPackageIntoSelf", "");
-    final PyFileImpl initPy = (PyFileImpl) myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("mygame/__init__.py"));
-    final PsiElement element = initPy.findExportedName("display");
-    assertInstanceOf(element, PyFile.class);
-    assertNotParsed(initPy);
-  }
-
   public void testVariableIndex() {
     getTestFile();
     GlobalSearchScope scope = GlobalSearchScope.allScope(myFixture.getProject());

python/testSrc/com/jetbrains/python/inspections/PyNoneFunctionAssignmentInspectionTest.java

@@ -27,6 +27,16 @@ public class PyNoneFunctionAssignmentInspectionTest extends PyTestCase {
     doTest();
   }
 
+  // PY-10883
+  public void testMethodWithInheritors() {
+    doTest();
+  }
+
+  // PY-10883
+  public void testDecoratedMethod() {
+    doTest();
+  }
+
   private void doTest() {
     myFixture.configureByFile("inspections/PyNoneFunctionAssignmentInspection/" + getTestName(true) + ".py");
     myFixture.enableInspections(PyNoneFunctionAssignmentInspection.class);

python/testSrc/com/jetbrains/python/inspections/PyUnusedImportTest.java

@@ -32,6 +32,11 @@ public class PyUnusedImportTest extends PyTestCase {
     doTest("test1.py");
   }
 
+  // PY-10667
+  public void testUsedLastImport() {
+    doTest("test1.py");
+  }
+
   private void doTest(@NotNull String filename) {
     myFixture.copyDirectoryToProject("inspections/unusedImport/" + getTestName(true), "");
     myFixture.enableInspections(PyUnresolvedReferencesInspection.class);