diff --git a/python/src/META-INF/python-core.xml b/python/src/META-INF/python-core.xml index d943f2319f0a..f6ef8f8287d4 100644 --- a/python/src/META-INF/python-core.xml +++ b/python/src/META-INF/python-core.xml @@ -96,6 +96,7 @@ + diff --git a/python/src/com/jetbrains/python/psi/PyUtil.java b/python/src/com/jetbrains/python/psi/PyUtil.java index f1edf4f93ce1..47e66d75db5a 100644 --- a/python/src/com/jetbrains/python/psi/PyUtil.java +++ b/python/src/com/jetbrains/python/psi/PyUtil.java @@ -69,7 +69,9 @@ import com.jetbrains.python.psi.impl.PyBuiltinCache; import com.jetbrains.python.psi.impl.PyPsiUtils; import com.jetbrains.python.psi.impl.PythonLanguageLevelPusher; import com.jetbrains.python.psi.resolve.PyResolveContext; +import com.jetbrains.python.psi.resolve.QualifiedNameFinder; import com.jetbrains.python.psi.resolve.RatedResolveResult; +import com.jetbrains.python.psi.stubs.PySetuptoolsNamespaceIndex; import com.jetbrains.python.psi.types.*; import com.jetbrains.python.refactoring.classes.PyDependenciesComparator; import com.jetbrains.python.refactoring.classes.extractSuperclass.PyExtractSuperclassHelper; @@ -1008,7 +1010,7 @@ public class PyUtil { if (level.isAtLeast(LanguageLevel.PYTHON33)) { return true; } - return hasNamespacePackageFile(directory); + return isSetuptoolsNamespacePackage(directory); } public static boolean isPackage(@NotNull PsiFile file) { @@ -1027,18 +1029,15 @@ public class PyUtil { return null; } - private static boolean hasNamespacePackageFile(@NotNull PsiDirectory directory) { - final String name = directory.getName().toLowerCase(); - final PsiDirectory parent = directory.getParent(); - if (parent != null) { - for (PsiFile file : parent.getFiles()) { - final String filename = file.getName().toLowerCase(); - if (filename.startsWith(name) && filename.endsWith("-nspkg.pth")) { - return true; - } - } - } - return false; + private static boolean isSetuptoolsNamespacePackage(@NotNull PsiDirectory directory) { + final String packagePath = getPackagePath(directory); + return packagePath != null && !PySetuptoolsNamespaceIndex.find(packagePath, directory.getProject()).isEmpty(); + } + + @Nullable + private static String getPackagePath(@NotNull PsiDirectory directory) { + final QualifiedName name = QualifiedNameFinder.findShortestImportableQName(directory); + return name != null ? name.toString() : null; } /** diff --git a/python/src/com/jetbrains/python/psi/stubs/PySetuptoolsNamespaceIndex.java b/python/src/com/jetbrains/python/psi/stubs/PySetuptoolsNamespaceIndex.java new file mode 100644 index 000000000000..2988922415d2 --- /dev/null +++ b/python/src/com/jetbrains/python/psi/stubs/PySetuptoolsNamespaceIndex.java @@ -0,0 +1,104 @@ +/* + * Copyright 2000-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jetbrains.python.psi.stubs; + +import com.intellij.openapi.project.Project; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.indexing.*; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.KeyDescriptor; +import com.jetbrains.python.psi.search.PyProjectScopeBuilder; +import org.jetbrains.annotations.NotNull; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author vlan + */ +public class PySetuptoolsNamespaceIndex extends ScalarIndexExtension { + public static final ID NAME = ID.create("Py.setuptools.namespace"); + private static final Pattern RE_NAMESPACE = Pattern.compile("sys\\.modules\\.setdefault\\('([^']*)'"); + private static final String NAMESPACE_FILE_SUFFIX = "-nspkg.pth"; + + private final EnumeratorStringDescriptor myKeyDescriptor = new EnumeratorStringDescriptor(); + + private final DataIndexer myDataIndexer = new DataIndexer() { + @NotNull + @Override + public Map map(@NotNull FileContent inputData) { + final CharSequence content = inputData.getContentAsText(); + final Matcher matcher = RE_NAMESPACE.matcher(content); + final Map results = new HashMap(); + while (matcher.find()) { + final String packageName = matcher.group(1); + results.put(packageName, null); + } + return results; + } + }; + + private FileBasedIndex.InputFilter myInputFilter = new FileBasedIndex.InputFilter() { + @Override + public boolean acceptInput(@NotNull VirtualFile file) { + return file.getName().endsWith(NAMESPACE_FILE_SUFFIX); + } + }; + + @NotNull + @Override + public ID getName() { + return NAME; + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return myDataIndexer; + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return myKeyDescriptor; + } + + @NotNull + @Override + public FileBasedIndex.InputFilter getInputFilter() { + return myInputFilter; + } + + @Override + public boolean dependsOnFileContent() { + return true; + } + + @Override + public int getVersion() { + return 0; + } + + @NotNull + public static Collection find(@NotNull String name, @NotNull Project project) { + final GlobalSearchScope scope = PyProjectScopeBuilder.excludeSdkTestsScope(project); + return FileBasedIndex.getInstance().getContainingFiles(NAME, name, scope); + } +} diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/a.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/a.py new file mode 100644 index 000000000000..4786a02c9c6a --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/a.py @@ -0,0 +1,5 @@ +from google.protobuf import service +from foo.bar import baz + +print(service.Service) +print(baz.Baz) diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foo/bar/__init__.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foo/bar/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foo/bar/baz.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foo/bar/baz.py new file mode 100644 index 000000000000..78292a636e5a --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foo/bar/baz.py @@ -0,0 +1,2 @@ +class Baz(object): + pass diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foobar-0.1-py2.7-nspkg.pth b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foobar-0.1-py2.7-nspkg.pth new file mode 100644 index 000000000000..90778b0afd58 --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/foobar-0.1-py2.7-nspkg.pth @@ -0,0 +1 @@ +import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('spam',));ie = os.path.exists(os.path.join(p,'__init__.py'));m = not ie and sys.modules.setdefault('spam', types.ModuleType('spam'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p) diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/google/protobuf/__init__.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/google/protobuf/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/google/protobuf/service.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/google/protobuf/service.py new file mode 100644 index 000000000000..cfca8c4ad5c1 --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/google/protobuf/service.py @@ -0,0 +1,2 @@ +class Service(object): + pass diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/protobuf-2.6.1-py2.7-nspkg.pth b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/protobuf-2.6.1-py2.7-nspkg.pth new file mode 100644 index 000000000000..6b67a7627412 --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NamespacePackageNameDoesntMatchFileName/protobuf-2.6.1-py2.7-nspkg.pth @@ -0,0 +1 @@ +import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('google',));ie = os.path.exists(os.path.join(p,'__init__.py'));m = not ie and sys.modules.setdefault('google', types.ModuleType('google'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p) diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/a.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/a.py new file mode 100644 index 000000000000..656f189a62c5 --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/a.py @@ -0,0 +1,4 @@ +from linkedin.utils.file import temp_directory + + +temp_directory() diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/linkedin/utils/file.py b/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/linkedin/utils/file.py new file mode 100644 index 000000000000..886432431626 --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/linkedin/utils/file.py @@ -0,0 +1,2 @@ +def temp_directory(): + pass diff --git a/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/zzz-lipy_utils-0.1.67-py2.6-nspkg.pth b/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/zzz-lipy_utils-0.1.67-py2.6-nspkg.pth new file mode 100644 index 000000000000..3353cab47921 --- /dev/null +++ b/python/testData/inspections/PyUnresolvedReferencesInspection/NestedNamespacePackageName/zzz-lipy_utils-0.1.67-py2.6-nspkg.pth @@ -0,0 +1,2 @@ +import sys,types,os,pkgutil; p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('linkedin',)); ie = os.path.exists(os.path.join(p,'__init__.py')); m = not ie and sys.modules.setdefault('linkedin',types.ModuleType('linkedin')); mp = (m or []) and m.__dict__.setdefault('__path__',[]); (p not in mp) and mp.append(p); mp[:] = m and pkgutil.extend_path(mp, 'linkedin') or mp +import sys,types,os,pkgutil; p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('linkedin', 'utils')); ie = os.path.exists(os.path.join(p,'__init__.py')); m = not ie and sys.modules.setdefault('linkedin.utils',types.ModuleType('linkedin.utils')); mp = (m or []) and m.__dict__.setdefault('__path__',[]); (p not in mp) and mp.append(p); mp[:] = m and pkgutil.extend_path(mp, 'linkedin.utils') or mp; m and __import__('linkedin') and setattr(sys.modules['linkedin'], 'utils', m) diff --git a/python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java b/python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java index 3451712f6bcd..9440d666c07f 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java @@ -386,6 +386,16 @@ public class PyUnresolvedReferencesInspectionTest extends PyInspectionTestCase { doMultiFileTest(); } + // PY-12738 + public void testNamespacePackageNameDoesntMatchFileName() { + doMultiFileTest(); + } + + // PY-13259 + public void testNestedNamespacePackageName() { + doMultiFileTest(); + } + @NotNull @Override protected Class getInspectionClass() {