diff --git a/python/src/com/jetbrains/extenstions/QualifiedNameExt.kt b/python/src/com/jetbrains/extenstions/QualifiedNameExt.kt index 706ede9fc3b6..900cb57142f3 100644 --- a/python/src/com/jetbrains/extenstions/QualifiedNameExt.kt +++ b/python/src/com/jetbrains/extenstions/QualifiedNameExt.kt @@ -142,7 +142,7 @@ fun QualifiedName.getElementAndResolvableName(context: QNameResolveContext, stop } if (element == null) { // Resolve against roots - element = resolveQualifiedNameWithClasses(currentName, resolveContext).firstOrNull() + element = resolveQualifiedName(currentName, resolveContext).firstOrNull() } if (element != null || stopOnFirstFail) { diff --git a/python/src/com/jetbrains/python/codeInsight/typing/PyStubPackages.kt b/python/src/com/jetbrains/python/codeInsight/typing/PyStubPackages.kt index 464d5f14dc82..b56c39a16bb3 100644 --- a/python/src/com/jetbrains/python/codeInsight/typing/PyStubPackages.kt +++ b/python/src/com/jetbrains/python/codeInsight/typing/PyStubPackages.kt @@ -4,7 +4,6 @@ package com.jetbrains.python.codeInsight.typing import com.intellij.openapi.module.Module -import com.intellij.openapi.module.ModuleUtilCore import com.intellij.openapi.project.Project import com.intellij.openapi.roots.ProjectFileIndex import com.intellij.openapi.util.Key @@ -18,6 +17,7 @@ import com.jetbrains.python.psi.PyFile import com.jetbrains.python.psi.PyUtil import com.jetbrains.python.psi.resolve.RatedResolveResult import com.jetbrains.python.pyi.PyiFile +import com.jetbrains.python.pyi.PyiUtil internal const val STUBS_SUFFIX = "-stubs" private val STUB_PACKAGE_KEY = Key("PY_STUB_PACKAGE") @@ -82,34 +82,6 @@ private fun doTransferStubPackageMarker(resolvedSubdir: PsiDirectory) { PyUtil.turnDirIntoInit(resolvedSubdir)?.putUserData(STUB_PACKAGE_KEY, true) } -/** - * Filters resolved elements according to their import priority in sys.path and - * [PEP 561](https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order) rules. - */ -fun filterTopPriorityResults(resolved: List, module: Module?): List { - if (resolved.isEmpty()) return emptyList() - - val groupedResults = resolved.groupByTo(sortedMapOf>()) { resolvedElementPriority(it, module) } - - if (groupedResults.containsKey(Priority.NAMESPACE_PACKAGE) && - groupedResults.headMap(Priority.NAMESPACE_PACKAGE).isEmpty()) return groupedResults[Priority.NAMESPACE_PACKAGE]!! - - groupedResults.remove(Priority.NAMESPACE_PACKAGE) - - return if (groupedResults.containsKey(Priority.STUB_PACKAGE) && groupedResults.headMap(Priority.STUB_PACKAGE).isEmpty()) { - // stub packages + next by priority - // because stub packages could be partial - - val stub = groupedResults[Priority.STUB_PACKAGE]!!.first() - val nextByPriority = groupedResults.tailMap(Priority.STUB_PACKAGE).values.asSequence().drop(1).take(1).flatten().firstOrNull() - - listOfNotNull(stub, nextByPriority) - } - else { - listOf(groupedResults.values.first().first()) - } -} - fun removeRuntimeModulesForWhomStubModulesFound(resolved: List): List { val stubPkgModules = mutableSetOf() @@ -135,55 +107,22 @@ private fun getClassOrContentOrSourceRoot(project: Project, file: VirtualFile): return null } -private fun isPyi(element: PsiElement) = element is PyiFile || PyUtil.turnDirIntoInit(element) is PyiFile - -private fun isNamespacePackage(element: PsiElement): Boolean { - if (element is PsiDirectory) { - val level = PyUtil.getLanguageLevelForVirtualFile(element.project, element.virtualFile) - if (!level.isPython2) { - return PyUtil.turnDirIntoInit(element) == null - } - } - return false -} - -/** - * See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order]. - */ -private fun resolvedElementPriority(element: PsiElement, module: Module?) = when { - isNamespacePackage(element) -> Priority.NAMESPACE_PACKAGE - isUserFile(element, module) -> if (isPyi(element)) Priority.USER_STUB else Priority.USER_CODE - isInStubPackage(element) -> Priority.STUB_PACKAGE - isInTypeShed(element) -> Priority.TYPESHED - isPyi(element) -> Priority.PROVIDED_STUB - isInInlinePackage(element, module) -> Priority.INLINE_PACKAGE - else -> Priority.OTHER -} - -private fun isUserFile(element: PsiElement, module: Module?) = - module != null && - element is PsiFileSystemItem && - element.virtualFile.let { it != null && ModuleUtilCore.moduleContainsFile(module, it, false) } - /** * See [findStubPackage] and [transferStubPackageMarker]. */ fun isInStubPackage(element: PsiElement) = element.getUserData(STUB_PACKAGE_KEY) == true -private fun isInTypeShed(element: PsiElement) = - isPyi(element) && (element as? PsiFileSystemItem)?.virtualFile.let { it != null && PyTypeShed.isInside(it) } - /** * See [https://www.python.org/dev/peps/pep-0561/#packaging-type-information]. * Value is cached in element's user data. */ -private fun isInInlinePackage(element: PsiElement, module: Module?): Boolean { +internal fun isInInlinePackage(element: PsiElement, module: Module?): Boolean { if (module == null) return false val cached = element.getUserData(INLINE_PACKAGE_KEY) if (cached != null) return cached - val result = !isPyi(element) && (element is PyFile || PyUtil.turnDirIntoInit(element) is PyFile) && getPyTyped(element) != null + val result = !PyiUtil.isPyiFileOfPackage(element) && (element is PyFile || PyUtil.turnDirIntoInit(element) is PyFile) && getPyTyped(element) != null element.putUserData(INLINE_PACKAGE_KEY, result) return result @@ -208,19 +147,4 @@ private fun getPyTyped(element: PsiElement?): VirtualFile? { } return null -} - -/** - * See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order]. - * Order is important, see [filterTopPriorityResults]. - */ -private enum class Priority { - USER_STUB, // pyi file located in user's project - USER_CODE, // py file located in user's project - PROVIDED_STUB, // pyi file provided with installed lib and located inside it - STUB_PACKAGE, // pyi file located in some stub package - INLINE_PACKAGE, // py file located in some inline package - TYPESHED, // pyi file located in typeshed - OTHER, // other cases, e.g. py file located inside installed lib - NAMESPACE_PACKAGE // namespace package has the lowest priority } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt b/python/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt index cc76c8b12db1..df53b0252ef3 100644 --- a/python/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt +++ b/python/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt @@ -18,8 +18,7 @@ import com.intellij.psi.PsiElement import com.intellij.psi.PsiFileSystemItem import com.intellij.psi.PsiManager import com.intellij.psi.util.QualifiedName -import com.jetbrains.python.codeInsight.typing.PyTypeShed -import com.jetbrains.python.codeInsight.typing.filterTopPriorityResults +import com.jetbrains.python.codeInsight.typing.* import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil import com.jetbrains.python.facet.PythonPathContributingFacet import com.jetbrains.python.psi.LanguageLevel @@ -28,8 +27,10 @@ import com.jetbrains.python.psi.PyUtil import com.jetbrains.python.psi.impl.PyBuiltinCache import com.jetbrains.python.psi.impl.PyImportResolver import com.jetbrains.python.pyi.PyiFile +import com.jetbrains.python.pyi.PyiUtil import com.jetbrains.python.sdk.PySdkUtil import com.jetbrains.python.sdk.PythonSdkType +import java.util.regex.Pattern /** * Python resolve utilities for qualified names. @@ -39,18 +40,6 @@ import com.jetbrains.python.sdk.PythonSdkType * @author vlan */ -/** - * Resolves qualified [name] to the list of packages, modules and, sometimes, classes. - * - * This method does not take into account source roots order (see PY-28321 as an example). - * The sole purpose of the method is to support classes that require class resolution until they can be migrated to [resolveQualifiedName]. - * - * @see resolveQualifiedName - */ -@Deprecated("This method does not provide proper source root resolution") -fun resolveQualifiedNameWithClasses(name: QualifiedName, context: PyQualifiedNameResolveContext): List { - return resolveQualifiedName(name, context, ::resultsFromRoots) -} /** * Resolves a qualified [name] to the list of packages and modules according to the [context]. @@ -58,12 +47,6 @@ fun resolveQualifiedNameWithClasses(name: QualifiedName, context: PyQualifiedNam * @see resolveTopLevelMember */ fun resolveQualifiedName(name: QualifiedName, context: PyQualifiedNameResolveContext): List { - return resolveQualifiedName(name, context, ::resolveModuleFromRoots) -} - -private fun resolveQualifiedName(name: QualifiedName, - context: PyQualifiedNameResolveContext, - resolveFromRoots: (QualifiedName, PyQualifiedNameResolveContext) -> List): List { checkAccess() if (!context.isValid) { return emptyList() @@ -87,8 +70,7 @@ private fun resolveQualifiedName(name: QualifiedName, val foreignResults = foreignResults(name, context) val pythonResults = listOf(relativeResults, - // TODO: replace with resolveFromRoots when namespace package magic features PY-16688, PY-23087 are implemented - resultsFromRoots(name, context), + resolveModuleFromRoots(name, context), relativeResultsFromSkeletons(name, context)).flatten().distinct() val allResults = pythonResults + foreignResults val results = if (name.componentCount > 0) filterTopPriorityResults(pythonResults, context.module) + foreignResults else allResults @@ -106,7 +88,7 @@ private fun resolveQualifiedName(name: QualifiedName, private fun resolveModuleFromRoots(name: QualifiedName, context: PyQualifiedNameResolveContext): List { val head = name.removeTail(name.componentCount - 1) val nameNoHead = name.removeHead(1) - return nameNoHead.components.fold(resultsFromRoots(head, context)) { results, component -> + return nameNoHead.components.fold(resultsFromRoots(head, context).distinct()) { results, component -> filterTopPriorityResults(results, context.module) .asSequence() .filterIsInstance() @@ -115,6 +97,7 @@ private fun resolveModuleFromRoots(name: QualifiedName, context: PyQualifiedName } } + /** * Resolves a [name] to the first module member defined at the top-level. */ @@ -341,3 +324,90 @@ private fun isRelativeImportResult(name: QualifiedName, directory: PsiDirectory, private fun checkAccess() { Preconditions.checkState(ApplicationManager.getApplication().isReadAccessAllowed, "This method requires read access") } + +/** + * Filters resolved elements according to their import priority in sys.path and + * [PEP 561](https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order) rules. + */ +private fun filterTopPriorityResults(resolved: List, module: Module?): List { + if (resolved.isEmpty()) return emptyList() + + val groupedResults = resolved.groupByTo(sortedMapOf>()) { resolvedElementPriority(it, module) } + + if (groupedResults.containsKey(Priority.NAMESPACE_PACKAGE) && + groupedResults.headMap(Priority.NAMESPACE_PACKAGE).isEmpty()) return groupedResults[Priority.NAMESPACE_PACKAGE]!! + + groupedResults.remove(Priority.NAMESPACE_PACKAGE) + + return if (groupedResults.containsKey(Priority.STUB_PACKAGE) && groupedResults.headMap(Priority.STUB_PACKAGE).isEmpty()) { + // stub packages + next by priority + // because stub packages could be partial + + val stub = groupedResults[Priority.STUB_PACKAGE]!!.first() + val nextByPriority = groupedResults.tailMap(Priority.STUB_PACKAGE).values.asSequence().drop(1).take(1).flatten().firstOrNull() + + listOfNotNull(stub, nextByPriority) + } + else { + listOf(groupedResults.values.first().first()) + } +} + +/** + * See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order]. + */ +private fun resolvedElementPriority(element: PsiElement, module: Module?) = when { + isNamespacePackage(element) -> Priority.NAMESPACE_PACKAGE + isUserFile(element, module) -> if (PyiUtil.isPyiFileOfPackage(element)) Priority.USER_STUB else Priority.USER_CODE + isInStubPackage(element) -> Priority.STUB_PACKAGE + isInTypeShed(element) -> Priority.TYPESHED + PyiUtil.isPyiFileOfPackage(element) -> Priority.PROVIDED_STUB + isInInlinePackage(element, module) -> Priority.INLINE_PACKAGE + else -> Priority.OTHER +} + +private fun isNamespacePackage(element: PsiElement): Boolean { + if (element is PsiDirectory) { + val level = PyUtil.getLanguageLevelForVirtualFile(element.project, element.virtualFile) + val initFile = PyUtil.turnDirIntoInit(element) ?: return !level.isPython2 + val initText = initFile.text ?: return false + return isNamespaceDeclaration(initText, pkgutilsInitPatterns) || isNamespaceDeclaration(initText, pkgResourcesInitPatterns) + } + return false +} + +private val pkgResourcesInitPatterns = listOf(Pattern.compile("^__import__\\(['\"]pkg_resources['\"]\\).declare_namespace\\(__name__\\)\\s*\$"), + Pattern.compile("^from pkg_resources import declare_namespace\\Rdeclare_namespace\\(__name__\\)\\s*\$"), + Pattern.compile("^import pkg_resources\\Rpkg_resources.declare_namespace\\(__name__\\)\\s*\$")) + + +private val pkgutilsInitPatterns = listOf(Pattern.compile("^__path__[ ]?=[ ]?__import__\\(['\"]pkgutil['\"]\\).extend_path\\(__path__, __name__\\)\\s*\$"), + Pattern.compile("^from pkgutil import extend_path\\R__path__[ ]?=[ ]?extend_path\\(__path__,[ ]?__name__\\)\\s*\$"), + Pattern.compile("^import pkgutil\\R__path__[ ]?=[ ]?pkgutil\\.extend_path\\(__path__,[ ]?__name__\\)\\s*\$")) + +private fun isNamespaceDeclaration(text: String, initTextPatterns: List): Boolean = + initTextPatterns.asSequence().map { it.matcher(text) }.any { it.matches() } + + +private fun isUserFile(element: PsiElement, module: Module?) = + module != null && + element is PsiFileSystemItem && + element.virtualFile.let { it != null && ModuleUtilCore.moduleContainsFile(module, it, false) } + +private fun isInTypeShed(element: PsiElement) = + PyiUtil.isPyiFileOfPackage(element) && (element as? PsiFileSystemItem)?.virtualFile.let { it != null && PyTypeShed.isInside(it) } + +/** + * See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order]. + * Order is important, see [filterTopPriorityResults]. + */ +private enum class Priority { + USER_STUB, // pyi file located in user's project + USER_CODE, // py file located in user's project + PROVIDED_STUB, // pyi file provided with installed lib and located inside it + STUB_PACKAGE, // pyi file located in some stub package + INLINE_PACKAGE, // py file located in some inline package + TYPESHED, // pyi file located in typeshed + OTHER, // other cases, e.g. py file located inside installed lib + NAMESPACE_PACKAGE // namespace packag e has the lowest priority +} diff --git a/python/src/com/jetbrains/python/pyi/PyiUtil.java b/python/src/com/jetbrains/python/pyi/PyiUtil.java index 2905e32c2339..232bbfd0ea35 100644 --- a/python/src/com/jetbrains/python/pyi/PyiUtil.java +++ b/python/src/com/jetbrains/python/pyi/PyiUtil.java @@ -116,6 +116,10 @@ public class PyiUtil { return ObjectUtils.notNull(PyUtil.as(getOriginalElement(element), cls), element); } + public static boolean isPyiFileOfPackage(@NotNull PsiElement element) { + return element instanceof PyiFile || PyUtil.turnDirIntoInit(element) instanceof PyiFile; + } + private static boolean pyButNotPyiFile(@Nullable PsiFile file) { return file instanceof PyFile && !(file instanceof PyiFile); } diff --git a/python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/__init__.py b/python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/__init__.py new file mode 100644 index 000000000000..ece379ce2fea --- /dev/null +++ b/python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/__init__.py @@ -0,0 +1,2 @@ +import pkg_resources +pkg_resources.declare_namespace(__name__) diff --git a/python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/a.py b/python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/a.py new file mode 100644 index 000000000000..c394838ae54d --- /dev/null +++ b/python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/a.py @@ -0,0 +1,2 @@ +import pkg.second +# diff --git a/python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/__init__.py b/python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/__init__.py new file mode 100644 index 000000000000..de40ea7ca058 --- /dev/null +++ b/python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/second.py b/python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/second.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/__init__.py b/python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/__init__.py new file mode 100644 index 000000000000..0260537a02bb --- /dev/null +++ b/python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/a.py b/python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/a.py new file mode 100644 index 000000000000..c394838ae54d --- /dev/null +++ b/python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/a.py @@ -0,0 +1,2 @@ +import pkg.second +# diff --git a/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py b/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py new file mode 100644 index 000000000000..0bfb5a62b4f6 --- /dev/null +++ b/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) \ No newline at end of file diff --git a/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/second.py b/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/second.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java b/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java index a667c95df8c0..0c7e0ff7bef0 100644 --- a/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java +++ b/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java @@ -466,8 +466,7 @@ public class PyMultiFileResolveTest extends PyMultiFileResolveTestCase { } // PY-28321 - // TODO: The test should be turned on as soon as PY-16688 and PY-23087 are implemented - public void ignoreTestImportManySourceRootsReverseRootOrder() { + public void testImportManySourceRootsReverseRootOrder() { myFixture.copyDirectoryToProject("importManySourceRoots", ""); runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> { final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py"); @@ -478,6 +477,38 @@ public class PyMultiFileResolveTest extends PyMultiFileResolveTestCase { }); } + // PY-16688 + public void testPkgResourcesNamespace() { + myFixture.copyDirectoryToProject("pkgResourcesNamespace", ""); + runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> { + final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py"); + final PsiReference ref = PyResolveTestCase.findReferenceByMarker(psiFile); + assertInstanceOf(ref, PsiPolyVariantReference.class); + final List elements = PyUtil.multiResolveTopPriority((PsiPolyVariantReference)ref); + assertEquals(1, elements.size()); + PsiFile root1 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root1/pkg/__init__.py")); + PsiFile root2 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root2/pkg/__init__.py")); + assertNotParsed(root1); + assertNotParsed(root2); + }); + } + + // PY-23087 + public void testPkgutilNamespace() { + myFixture.copyDirectoryToProject("pkgutilNamespace", ""); + runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> { + final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py"); + final PsiReference ref = PyResolveTestCase.findReferenceByMarker(psiFile); + assertInstanceOf(ref, PsiPolyVariantReference.class); + final List elements = PyUtil.multiResolveTopPriority((PsiPolyVariantReference)ref); + assertEquals(1, elements.size()); + PsiFile root1 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root1/pkg/__init__.py")); + PsiFile root2 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root2/pkg/__init__.py")); + assertNotParsed(root1); + assertNotParsed(root2); + }); + } + // PY-22522 public void testBothForeignAndSourceRootImportResultsReturned() { myFixture.copyDirectoryToProject("bothForeignAndSourceRootImportResultsReturned", "");