Extract setuptools namespace package names from *-nspkg.pth files (PY-12738, PY-13262, PY-13259)

python/src/META-INF/python-core.xml

@@ -96,6 +96,7 @@
     <stubIndex implementation="com.jetbrains.python.psi.stubs.PyInstanceAttributeIndex"/>
     <stubIndex implementation="com.jetbrains.python.psi.stubs.PyDecoratorStubIndex"/>
     <fileBasedIndex implementation="com.jetbrains.python.psi.stubs.PyModuleNameIndex"/>
+    <fileBasedIndex implementation="com.jetbrains.python.psi.stubs.PySetuptoolsNamespaceIndex"/>
 
     <declarationRangeHandler key="com.jetbrains.python.psi.PyClass"
                              implementationClass="com.jetbrains.python.codeInsight.PyDeclarationRangeHandler"/>

python/src/com/jetbrains/python/psi/PyUtil.java

@@ -69,7 +69,9 @@ import com.jetbrains.python.psi.impl.PyBuiltinCache;
 import com.jetbrains.python.psi.impl.PyPsiUtils;
 import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher;
 import com.jetbrains.python.psi.resolve.PyResolveContext;
+import com.jetbrains.python.psi.resolve.QualifiedNameFinder;
 import com.jetbrains.python.psi.resolve.RatedResolveResult;
+import com.jetbrains.python.psi.stubs.PySetuptoolsNamespaceIndex;
 import com.jetbrains.python.psi.types.*;
 import com.jetbrains.python.refactoring.classes.PyDependenciesComparator;
 import com.jetbrains.python.refactoring.classes.extractSuperclass.PyExtractSuperclassHelper;
@@ -1008,7 +1010,7 @@ public class PyUtil {
     if (level.isAtLeast(LanguageLevel.PYTHON33)) {
       return true;
     }
-    return hasNamespacePackageFile(directory);
+    return isSetuptoolsNamespacePackage(directory);
   }
 
   public static boolean isPackage(@NotNull PsiFile file) {
@@ -1027,18 +1029,15 @@ public class PyUtil {
     return null;
   }
 
-  private static boolean hasNamespacePackageFile(@NotNull PsiDirectory directory) {
-    final String name = directory.getName().toLowerCase();
-    final PsiDirectory parent = directory.getParent();
-    if (parent != null) {
-      for (PsiFile file : parent.getFiles()) {
-        final String filename = file.getName().toLowerCase();
-        if (filename.startsWith(name) && filename.endsWith("-nspkg.pth")) {
-          return true;
-        }
-      }
-    }
-    return false;
+  private static boolean isSetuptoolsNamespacePackage(@NotNull PsiDirectory directory) {
+    final String packagePath = getPackagePath(directory);
+    return packagePath != null && !PySetuptoolsNamespaceIndex.find(packagePath, directory.getProject()).isEmpty();
+  }
+
+  @Nullable
+  private static String getPackagePath(@NotNull PsiDirectory directory) {
+    final QualifiedName name = QualifiedNameFinder.findShortestImportableQName(directory);
+    return name != null ? name.toString() : null;
   }
 
   /**

python/src/com/jetbrains/python/psi/stubs/PySetuptoolsNamespaceIndex.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2000-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.psi.stubs;
+
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.search.GlobalSearchScope;
+import com.intellij.util.indexing.*;
+import com.intellij.util.io.EnumeratorStringDescriptor;
+import com.intellij.util.io.KeyDescriptor;
+import com.jetbrains.python.psi.search.PyProjectScopeBuilder;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author vlan
+ */
+public class PySetuptoolsNamespaceIndex extends ScalarIndexExtension<String> {
+  public static final ID<String, Void> NAME = ID.create("Py.setuptools.namespace");
+  private static final Pattern RE_NAMESPACE = Pattern.compile("sys\\.modules\\.setdefault\\('([^']*)'");
+  private static final String NAMESPACE_FILE_SUFFIX = "-nspkg.pth";
+
+  private final EnumeratorStringDescriptor myKeyDescriptor = new EnumeratorStringDescriptor();
+
+  private final DataIndexer<String, Void, FileContent> myDataIndexer = new DataIndexer<String, Void, FileContent>() {
+    @NotNull
+    @Override
+    public Map<String, Void> map(@NotNull FileContent inputData) {
+      final CharSequence content = inputData.getContentAsText();
+      final Matcher matcher = RE_NAMESPACE.matcher(content);
+      final Map<String, Void> results = new HashMap<String, Void>();
+      while (matcher.find()) {
+        final String packageName = matcher.group(1);
+        results.put(packageName, null);
+      }
+      return results;
+    }
+  };
+
+  private FileBasedIndex.InputFilter myInputFilter = new FileBasedIndex.InputFilter() {
+    @Override
+    public boolean acceptInput(@NotNull VirtualFile file) {
+      return file.getName().endsWith(NAMESPACE_FILE_SUFFIX);
+    }
+  };
+
+  @NotNull
+  @Override
+  public ID<String, Void> getName() {
+    return NAME;
+  }
+
+  @NotNull
+  @Override
+  public DataIndexer<String, Void, FileContent> getIndexer() {
+    return myDataIndexer;
+  }
+
+  @NotNull
+  @Override
+  public KeyDescriptor<String> getKeyDescriptor() {
+    return myKeyDescriptor;
+  }
+
+  @NotNull
+  @Override
+  public FileBasedIndex.InputFilter getInputFilter() {
+    return myInputFilter;
+  }
+
+  @Override
+  public boolean dependsOnFileContent() {
+    return true;
+  }
+
+  @Override
+  public int getVersion() {
+    return 0;
+  }
+
+  @NotNull
+  public static Collection<VirtualFile> find(@NotNull String name, @NotNull Project project) {
+    final GlobalSearchScope scope = PyProjectScopeBuilder.excludeSdkTestsScope(project);
+    return FileBasedIndex.getInstance().getContainingFiles(NAME, name, scope);
+  }
+}

python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/a.py

@@ -0,0 +1,5 @@
+from google.protobuf import service
+from <error descr="Unresolved reference 'foo'">foo</error>.<warning descr="Unresolved reference 'bar'">bar</warning> import <error descr="Unresolved reference 'baz'">baz</error>
+
+print(service.Service)
+print(baz.Baz)

python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foo/bar/baz.py

@@ -0,0 +1,2 @@
+class Baz(object):
+    pass

python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foobar-0.1-py2.7-nspkg.pth

@@ -0,0 +1 @@
+import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('spam',));ie = os.path.exists(os.path.join(p,'__init__.py'));m = not ie and sys.modules.setdefault('spam', types.ModuleType('spam'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)

python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/google/protobuf/service.py

@@ -0,0 +1,2 @@
+class Service(object):
+    pass

python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/protobuf-2.6.1-py2.7-nspkg.pth

@@ -0,0 +1 @@
+import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('google',));ie = os.path.exists(os.path.join(p,'__init__.py'));m = not ie and sys.modules.setdefault('google', types.ModuleType('google'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)

python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/a.py

@@ -0,0 +1,4 @@
+from linkedin.utils.file import temp_directory
+
+
+temp_directory()

python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/linkedin/utils/file.py

@@ -0,0 +1,2 @@
+def temp_directory():
+    pass

python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/zzz-lipy_utils-0.1.67-py2.6-nspkg.pth

@@ -0,0 +1,2 @@
+import sys,types,os,pkgutil; p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('linkedin',)); ie = os.path.exists(os.path.join(p,'__init__.py')); m = not ie and sys.modules.setdefault('linkedin',types.ModuleType('linkedin')); mp = (m or []) and m.__dict__.setdefault('__path__',[]); (p not in mp) and mp.append(p); mp[:] = m and pkgutil.extend_path(mp, 'linkedin') or mp
+import sys,types,os,pkgutil; p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('linkedin', 'utils')); ie = os.path.exists(os.path.join(p,'__init__.py')); m = not ie and sys.modules.setdefault('linkedin.utils',types.ModuleType('linkedin.utils')); mp = (m or []) and m.__dict__.setdefault('__path__',[]); (p not in mp) and mp.append(p); mp[:] = m and pkgutil.extend_path(mp, 'linkedin.utils') or mp; m and __import__('linkedin') and setattr(sys.modules['linkedin'], 'utils', m)

python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java

@@ -386,6 +386,16 @@ public class PyUnresolvedReferencesInspectionTest extends PyInspectionTestCase {
     doMultiFileTest();
   }
 
+  // PY-12738
+  public void testNamespacePackageNameDoesntMatchFileName() {
+    doMultiFileTest();
+  }
+
+  // PY-13259
+  public void testNestedNamespacePackageName() {
+    doMultiFileTest();
+  }
+
   @NotNull
   @Override
   protected Class<? extends PyInspection> getInspectionClass() {