MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-13 21:55:01 +07:00
Code Issues Packages Projects Releases Wiki Activity

support namespace packages defined via pkg_resources and pkg_util (PY-16688, PY-23087)

Browse Source 
GitOrigin-RevId: fc979a18d3c8bf310f6ae4381ac7f365e8b6c180
This commit is contained in:
Aleksei Kniazev
2019-09-09 10:02:33 +00:00
committed by intellij-monorepo-bot
parent 79515c9d19
commit fbfef48442
13 changed files with 144 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/extenstions/QualifiedNameExt.kt
+1 -1
View File
@@ -142,7 +142,7 @@ fun QualifiedName.getElementAndResolvableName(context: QNameResolveContext, stop
}
if (element == null) { // Resolve against roots
element = resolveQualifiedNameWithClasses(currentName, resolveContext).firstOrNull()
element = resolveQualifiedName(currentName, resolveContext).firstOrNull()
}
if (element != null || stopOnFirstFail) {
python/src/com/jetbrains/python/codeInsight/typing/PyStubPackages.kt
+3 -79
View File
@@ -4,7 +4,6 @@
package com.jetbrains.python.codeInsight.typing
import com.intellij.openapi.module.Module
import com.intellij.openapi.module.ModuleUtilCore
import com.intellij.openapi.project.Project
import com.intellij.openapi.roots.ProjectFileIndex
import com.intellij.openapi.util.Key
@@ -18,6 +17,7 @@ import com.jetbrains.python.psi.PyFile
import com.jetbrains.python.psi.PyUtil
import com.jetbrains.python.psi.resolve.RatedResolveResult
import com.jetbrains.python.pyi.PyiFile
import com.jetbrains.python.pyi.PyiUtil
internal const val STUBS_SUFFIX = "-stubs"
private val STUB_PACKAGE_KEY = Key<Boolean>("PY_STUB_PACKAGE")
@@ -82,34 +82,6 @@ private fun doTransferStubPackageMarker(resolvedSubdir: PsiDirectory) {
PyUtil.turnDirIntoInit(resolvedSubdir)?.putUserData(STUB_PACKAGE_KEY, true)
}
/**
* Filters resolved elements according to their import priority in sys.path and
* [PEP 561](https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order) rules.
*/
fun filterTopPriorityResults(resolved: List<PsiElement>, module: Module?): List<PsiElement> {
if (resolved.isEmpty()) return emptyList()
val groupedResults = resolved.groupByTo(sortedMapOf<Priority, MutableList<PsiElement>>()) { resolvedElementPriority(it, module) }
if (groupedResults.containsKey(Priority.NAMESPACE_PACKAGE) &&
groupedResults.headMap(Priority.NAMESPACE_PACKAGE).isEmpty()) return groupedResults[Priority.NAMESPACE_PACKAGE]!!
groupedResults.remove(Priority.NAMESPACE_PACKAGE)
return if (groupedResults.containsKey(Priority.STUB_PACKAGE) && groupedResults.headMap(Priority.STUB_PACKAGE).isEmpty()) {
// stub packages + next by priority
// because stub packages could be partial
val stub = groupedResults[Priority.STUB_PACKAGE]!!.first()
val nextByPriority = groupedResults.tailMap(Priority.STUB_PACKAGE).values.asSequence().drop(1).take(1).flatten().firstOrNull()
listOfNotNull(stub, nextByPriority)
}
else {
listOf(groupedResults.values.first().first())
}
}
fun removeRuntimeModulesForWhomStubModulesFound(resolved: List<RatedResolveResult>): List<RatedResolveResult> {
val stubPkgModules = mutableSetOf<String>()
@@ -135,55 +107,22 @@ private fun getClassOrContentOrSourceRoot(project: Project, file: VirtualFile):
return null
}
private fun isPyi(element: PsiElement) = element is PyiFile || PyUtil.turnDirIntoInit(element) is PyiFile
private fun isNamespacePackage(element: PsiElement): Boolean {
if (element is PsiDirectory) {
val level = PyUtil.getLanguageLevelForVirtualFile(element.project, element.virtualFile)
if (!level.isPython2) {
return PyUtil.turnDirIntoInit(element) == null
}
}
return false
}
/**
* See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order].
*/
private fun resolvedElementPriority(element: PsiElement, module: Module?) = when {
isNamespacePackage(element) -> Priority.NAMESPACE_PACKAGE
isUserFile(element, module) -> if (isPyi(element)) Priority.USER_STUB else Priority.USER_CODE
isInStubPackage(element) -> Priority.STUB_PACKAGE
isInTypeShed(element) -> Priority.TYPESHED
isPyi(element) -> Priority.PROVIDED_STUB
isInInlinePackage(element, module) -> Priority.INLINE_PACKAGE
else -> Priority.OTHER
}
private fun isUserFile(element: PsiElement, module: Module?) =
module != null &&
element is PsiFileSystemItem &&
element.virtualFile.let { it != null && ModuleUtilCore.moduleContainsFile(module, it, false) }
/**
* See [findStubPackage] and [transferStubPackageMarker].
*/
fun isInStubPackage(element: PsiElement) = element.getUserData(STUB_PACKAGE_KEY) == true
private fun isInTypeShed(element: PsiElement) =
isPyi(element) && (element as? PsiFileSystemItem)?.virtualFile.let { it != null && PyTypeShed.isInside(it) }
/**
* See [https://www.python.org/dev/peps/pep-0561/#packaging-type-information].
* Value is cached in element's user data.
*/
private fun isInInlinePackage(element: PsiElement, module: Module?): Boolean {
internal fun isInInlinePackage(element: PsiElement, module: Module?): Boolean {
if (module == null) return false
val cached = element.getUserData(INLINE_PACKAGE_KEY)
if (cached != null) return cached
val result = !isPyi(element) && (element is PyFile || PyUtil.turnDirIntoInit(element) is PyFile) && getPyTyped(element) != null
val result = !PyiUtil.isPyiFileOfPackage(element) && (element is PyFile || PyUtil.turnDirIntoInit(element) is PyFile) && getPyTyped(element) != null
element.putUserData(INLINE_PACKAGE_KEY, result)
return result
@@ -208,19 +147,4 @@ private fun getPyTyped(element: PsiElement?): VirtualFile? {
}
return null
}
/**
* See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order].
* Order is important, see [filterTopPriorityResults].
*/
private enum class Priority {
USER_STUB, // pyi file located in user's project
USER_CODE, // py file located in user's project
PROVIDED_STUB, // pyi file provided with installed lib and located inside it
STUB_PACKAGE, // pyi file located in some stub package
INLINE_PACKAGE, // py file located in some inline package
TYPESHED, // pyi file located in typeshed
OTHER, // other cases, e.g. py file located inside installed lib
NAMESPACE_PACKAGE // namespace package has the lowest priority
}
python/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt
+93 -23
View File
@@ -18,8 +18,7 @@ import com.intellij.psi.PsiElement
import com.intellij.psi.PsiFileSystemItem
import com.intellij.psi.PsiManager
import com.intellij.psi.util.QualifiedName
import com.jetbrains.python.codeInsight.typing.PyTypeShed
import com.jetbrains.python.codeInsight.typing.filterTopPriorityResults
import com.jetbrains.python.codeInsight.typing.*
import com.jetbrains.python.codeInsight.userSkeletons.PyUserSkeletonsUtil
import com.jetbrains.python.facet.PythonPathContributingFacet
import com.jetbrains.python.psi.LanguageLevel
@@ -28,8 +27,10 @@ import com.jetbrains.python.psi.PyUtil
import com.jetbrains.python.psi.impl.PyBuiltinCache
import com.jetbrains.python.psi.impl.PyImportResolver
import com.jetbrains.python.pyi.PyiFile
import com.jetbrains.python.pyi.PyiUtil
import com.jetbrains.python.sdk.PySdkUtil
import com.jetbrains.python.sdk.PythonSdkType
import java.util.regex.Pattern
/**
* Python resolve utilities for qualified names.
@@ -39,18 +40,6 @@ import com.jetbrains.python.sdk.PythonSdkType
* @author vlan
*/
/**
* Resolves qualified [name] to the list of packages, modules and, sometimes, classes.
*
* This method does not take into account source roots order (see PY-28321 as an example).
* The sole purpose of the method is to support classes that require class resolution until they can be migrated to [resolveQualifiedName].
*
* @see resolveQualifiedName
*/
@Deprecated("This method does not provide proper source root resolution")
fun resolveQualifiedNameWithClasses(name: QualifiedName, context: PyQualifiedNameResolveContext): List<PsiElement> {
return resolveQualifiedName(name, context, ::resultsFromRoots)
}
/**
* Resolves a qualified [name] to the list of packages and modules according to the [context].
@@ -58,12 +47,6 @@ fun resolveQualifiedNameWithClasses(name: QualifiedName, context: PyQualifiedNam
* @see resolveTopLevelMember
*/
fun resolveQualifiedName(name: QualifiedName, context: PyQualifiedNameResolveContext): List<PsiElement> {
return resolveQualifiedName(name, context, ::resolveModuleFromRoots)
}
private fun resolveQualifiedName(name: QualifiedName,
context: PyQualifiedNameResolveContext,
resolveFromRoots: (QualifiedName, PyQualifiedNameResolveContext) -> List<PsiElement>): List<PsiElement> {
checkAccess()
if (!context.isValid) {
return emptyList()
@@ -87,8 +70,7 @@ private fun resolveQualifiedName(name: QualifiedName,
val foreignResults = foreignResults(name, context)
val pythonResults = listOf(relativeResults,
// TODO: replace with resolveFromRoots when namespace package magic features PY-16688, PY-23087 are implemented
resultsFromRoots(name, context),
resolveModuleFromRoots(name, context),
relativeResultsFromSkeletons(name, context)).flatten().distinct()
val allResults = pythonResults + foreignResults
val results = if (name.componentCount > 0) filterTopPriorityResults(pythonResults, context.module) + foreignResults else allResults
@@ -106,7 +88,7 @@ private fun resolveQualifiedName(name: QualifiedName,
private fun resolveModuleFromRoots(name: QualifiedName, context: PyQualifiedNameResolveContext): List<PsiElement> {
val head = name.removeTail(name.componentCount - 1)
val nameNoHead = name.removeHead(1)
return nameNoHead.components.fold(resultsFromRoots(head, context)) { results, component ->
return nameNoHead.components.fold(resultsFromRoots(head, context).distinct()) { results, component ->
filterTopPriorityResults(results, context.module)
.asSequence()
.filterIsInstance<PsiFileSystemItem>()
@@ -115,6 +97,7 @@ private fun resolveModuleFromRoots(name: QualifiedName, context: PyQualifiedName
}
}
/**
* Resolves a [name] to the first module member defined at the top-level.
*/
@@ -341,3 +324,90 @@ private fun isRelativeImportResult(name: QualifiedName, directory: PsiDirectory,
private fun checkAccess() {
Preconditions.checkState(ApplicationManager.getApplication().isReadAccessAllowed, "This method requires read access")
}
/**
* Filters resolved elements according to their import priority in sys.path and
* [PEP 561](https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order) rules.
*/
private fun filterTopPriorityResults(resolved: List<PsiElement>, module: Module?): List<PsiElement> {
if (resolved.isEmpty()) return emptyList()
val groupedResults = resolved.groupByTo(sortedMapOf<Priority, MutableList<PsiElement>>()) { resolvedElementPriority(it, module) }
if (groupedResults.containsKey(Priority.NAMESPACE_PACKAGE) &&
groupedResults.headMap(Priority.NAMESPACE_PACKAGE).isEmpty()) return groupedResults[Priority.NAMESPACE_PACKAGE]!!
groupedResults.remove(Priority.NAMESPACE_PACKAGE)
return if (groupedResults.containsKey(Priority.STUB_PACKAGE) && groupedResults.headMap(Priority.STUB_PACKAGE).isEmpty()) {
// stub packages + next by priority
// because stub packages could be partial
val stub = groupedResults[Priority.STUB_PACKAGE]!!.first()
val nextByPriority = groupedResults.tailMap(Priority.STUB_PACKAGE).values.asSequence().drop(1).take(1).flatten().firstOrNull()
listOfNotNull(stub, nextByPriority)
}
else {
listOf(groupedResults.values.first().first())
}
}
/**
* See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order].
*/
private fun resolvedElementPriority(element: PsiElement, module: Module?) = when {
isNamespacePackage(element) -> Priority.NAMESPACE_PACKAGE
isUserFile(element, module) -> if (PyiUtil.isPyiFileOfPackage(element)) Priority.USER_STUB else Priority.USER_CODE
isInStubPackage(element) -> Priority.STUB_PACKAGE
isInTypeShed(element) -> Priority.TYPESHED
PyiUtil.isPyiFileOfPackage(element) -> Priority.PROVIDED_STUB
isInInlinePackage(element, module) -> Priority.INLINE_PACKAGE
else -> Priority.OTHER
}
private fun isNamespacePackage(element: PsiElement): Boolean {
if (element is PsiDirectory) {
val level = PyUtil.getLanguageLevelForVirtualFile(element.project, element.virtualFile)
val initFile = PyUtil.turnDirIntoInit(element) ?: return !level.isPython2
val initText = initFile.text ?: return false
return isNamespaceDeclaration(initText, pkgutilsInitPatterns) || isNamespaceDeclaration(initText, pkgResourcesInitPatterns)
}
return false
}
private val pkgResourcesInitPatterns = listOf(Pattern.compile("^__import__\\(['\"]pkg_resources['\"]\\).declare_namespace\\(__name__\\)\\s*\$"),
Pattern.compile("^from pkg_resources import declare_namespace\\Rdeclare_namespace\\(__name__\\)\\s*\$"),
Pattern.compile("^import pkg_resources\\Rpkg_resources.declare_namespace\\(__name__\\)\\s*\$"))
private val pkgutilsInitPatterns = listOf(Pattern.compile("^__path__[ ]?=[ ]?__import__\\(['\"]pkgutil['\"]\\).extend_path\\(__path__, __name__\\)\\s*\$"),
Pattern.compile("^from pkgutil import extend_path\\R__path__[ ]?=[ ]?extend_path\\(__path__,[ ]?__name__\\)\\s*\$"),
Pattern.compile("^import pkgutil\\R__path__[ ]?=[ ]?pkgutil\\.extend_path\\(__path__,[ ]?__name__\\)\\s*\$"))
private fun isNamespaceDeclaration(text: String, initTextPatterns: List<Pattern>): Boolean =
initTextPatterns.asSequence().map { it.matcher(text) }.any { it.matches() }
private fun isUserFile(element: PsiElement, module: Module?) =
module != null &&
element is PsiFileSystemItem &&
element.virtualFile.let { it != null && ModuleUtilCore.moduleContainsFile(module, it, false) }
private fun isInTypeShed(element: PsiElement) =
PyiUtil.isPyiFileOfPackage(element) && (element as? PsiFileSystemItem)?.virtualFile.let { it != null && PyTypeShed.isInside(it) }
/**
* See [https://www.python.org/dev/peps/pep-0561/#type-checker-module-resolution-order].
* Order is important, see [filterTopPriorityResults].
*/
private enum class Priority {
USER_STUB, // pyi file located in user's project
USER_CODE, // py file located in user's project
PROVIDED_STUB, // pyi file provided with installed lib and located inside it
STUB_PACKAGE, // pyi file located in some stub package
INLINE_PACKAGE, // py file located in some inline package
TYPESHED, // pyi file located in typeshed
OTHER, // other cases, e.g. py file located inside installed lib
NAMESPACE_PACKAGE // namespace packag e has the lowest priority
}
python/src/com/jetbrains/python/pyi/PyiUtil.java
+4
View File
@@ -116,6 +116,10 @@ public class PyiUtil {
return ObjectUtils.notNull(PyUtil.as(getOriginalElement(element), cls), element);
}
public static boolean isPyiFileOfPackage(@NotNull PsiElement element) {
return element instanceof PyiFile || PyUtil.turnDirIntoInit(element) instanceof PyiFile;
}
private static boolean pyButNotPyiFile(@Nullable PsiFile file) {
return file instanceof PyFile && !(file instanceof PyiFile);
}
python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/__init__.py
+2
View File
@@ -0,0 +1,2 @@
import pkg_resources
pkg_resources.declare_namespace(__name__)
python/testData/resolve/multiFile/pkgResourcesNamespace/root1/pkg/a.py
+2
View File
@@ -0,0 +1,2 @@
import pkg.second
# <ref>
python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/__init__.py
+1
View File
@@ -0,0 +1 @@
__import__('pkg_resources').declare_namespace(__name__)
python/testData/resolve/multiFile/pkgResourcesNamespace/root2/pkg/second.py
View File
python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/__init__.py
+1
View File
@@ -0,0 +1 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
python/testData/resolve/multiFile/pkgutilNamespace/root1/pkg/a.py
+2
View File
@@ -0,0 +1,2 @@
import pkg.second
# <ref>
python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py
+2
View File
@@ -0,0 +1,2 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/second.py
View File
python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java
+33 -2
View File
@@ -466,8 +466,7 @@ public class PyMultiFileResolveTest extends PyMultiFileResolveTestCase {
}
// PY-28321
// TODO: The test should be turned on as soon as PY-16688 and PY-23087 are implemented
public void ignoreTestImportManySourceRootsReverseRootOrder() {
public void testImportManySourceRootsReverseRootOrder() {
myFixture.copyDirectoryToProject("importManySourceRoots", "");
runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> {
final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py");
@@ -478,6 +477,38 @@ public class PyMultiFileResolveTest extends PyMultiFileResolveTestCase {
});
}
// PY-16688
public void testPkgResourcesNamespace() {
myFixture.copyDirectoryToProject("pkgResourcesNamespace", "");
runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> {
final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py");
final PsiReference ref = PyResolveTestCase.findReferenceByMarker(psiFile);
assertInstanceOf(ref, PsiPolyVariantReference.class);
final List<PsiElement> elements = PyUtil.multiResolveTopPriority((PsiPolyVariantReference)ref);
assertEquals(1, elements.size());
PsiFile root1 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root1/pkg/__init__.py"));
PsiFile root2 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root2/pkg/__init__.py"));
assertNotParsed(root1);
assertNotParsed(root2);
});
}
// PY-23087
public void testPkgutilNamespace() {
myFixture.copyDirectoryToProject("pkgutilNamespace", "");
runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> {
final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py");
final PsiReference ref = PyResolveTestCase.findReferenceByMarker(psiFile);
assertInstanceOf(ref, PsiPolyVariantReference.class);
final List<PsiElement> elements = PyUtil.multiResolveTopPriority((PsiPolyVariantReference)ref);
assertEquals(1, elements.size());
PsiFile root1 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root1/pkg/__init__.py"));
PsiFile root2 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root2/pkg/__init__.py"));
assertNotParsed(root1);
assertNotParsed(root2);
});
}
// PY-22522
public void testBothForeignAndSourceRootImportResultsReturned() {
myFixture.copyDirectoryToProject("bothForeignAndSourceRootImportResultsReturned", "");