From a906dd493a6161c16a730c72480fc513adf482fd Mon Sep 17 00:00:00 2001 From: Aleksei Kniazev Date: Thu, 12 Dec 2019 17:39:03 +0300 Subject: [PATCH] account for comments and line breaks in __init__ files declaring namespace package (PY-38434) GitOrigin-RevId: 4a5a890113a8d8aa55053f6d96d946958476ea17 --- .../python/psi/resolve/PyResolveImportUtil.kt | 31 ++++++++++--------- .../pkgutilNamespace/root2/pkg/__init__.py | 4 +-- .../root1/pkg/__init__.py | 5 +++ .../root1/pkg/a.py | 2 ++ .../root2/pkg/__init__.py | 7 +++++ .../root2/pkg/second.py | 0 .../python/PyMultiFileResolveTest.java | 24 +++++++------- 7 files changed, 44 insertions(+), 29 deletions(-) create mode 100644 python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/__init__.py create mode 100644 python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/a.py create mode 100644 python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/__init__.py create mode 100644 python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/second.py diff --git a/python/python-psi-impl/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt b/python/python-psi-impl/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt index f0f32d04362c..d07e07273255 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt +++ b/python/python-psi-impl/src/com/jetbrains/python/psi/resolve/PyResolveImportUtil.kt @@ -408,25 +408,28 @@ private fun isNamespacePackage(element: PsiElement): Boolean { if (element is PsiDirectory) { val level = PyUtil.getLanguageLevelForVirtualFile(element.project, element.virtualFile) val initFile = PyUtil.turnDirIntoInit(element) ?: return !level.isPython2 - val initText = initFile.text ?: return false - return isNamespaceDeclaration(initText, pkgutilsInitPatterns) || isNamespaceDeclaration(initText, pkgResourcesInitPatterns) + val initLines = initFile.text.lineSequence() + .filterNot { line -> line.trim().let { it.isEmpty() || it.startsWith("#") } } + .take(2) + .toList() + return when (initLines.size) { + 1 -> oneLineNamespaceDeclarations.any { it.matcher(initLines.first()).matches() } + 2 -> multilineNamespaceDeclarations.any { it[0].matcher(initLines[0]).matches() && it[1].matcher(initLines[1]).matches() } + else -> false + } } return false } -private val pkgResourcesInitPatterns = listOf( - Pattern.compile("^__import__\\(['\"]pkg_resources['\"]\\).declare_namespace\\(__name__\\)\\s*\$"), - Pattern.compile("^from pkg_resources import declare_namespace\\Rdeclare_namespace\\(__name__\\)\\s*\$"), - Pattern.compile("^import pkg_resources\\Rpkg_resources.declare_namespace\\(__name__\\)\\s*\$")) +private val oneLineNamespaceDeclarations = listOf( + Pattern.compile("^__path__[ ]?=[ ]?__import__\\(['\"]pkgutil['\"]\\).extend_path\\(__path__, __name__\\).*"), + Pattern.compile("^__import__\\(['\"]pkg_resources['\"]\\).declare_namespace\\(__name__\\).*")) - -private val pkgutilsInitPatterns = listOf( - Pattern.compile("^__path__[ ]?=[ ]?__import__\\(['\"]pkgutil['\"]\\).extend_path\\(__path__, __name__\\)\\s*\$"), - Pattern.compile("^from pkgutil import extend_path\\R__path__[ ]?=[ ]?extend_path\\(__path__,[ ]?__name__\\)\\s*\$"), - Pattern.compile("^import pkgutil\\R__path__[ ]?=[ ]?pkgutil\\.extend_path\\(__path__,[ ]?__name__\\)\\s*\$")) - -private fun isNamespaceDeclaration(text: String, initTextPatterns: List): Boolean = - initTextPatterns.asSequence().map { it.matcher(text) }.any { it.matches() } +private val multilineNamespaceDeclarations = listOf( + listOf(Pattern.compile("^from pkgutil import extend_path.*"), Pattern.compile("^__path__[ ]?=[ ]?extend_path\\(__path__,[ ]?__name__\\).*")), + listOf(Pattern.compile("^import pkgutil.*"), Pattern.compile("^__path__[ ]?=[ ]?pkgutil\\.extend_path\\(__path__,[ ]?__name__\\).*")), + listOf(Pattern.compile("^from pkg_resources import declare_namespace.*"), Pattern.compile("^declare_namespace\\(__name__\\).*")), + listOf(Pattern.compile("^import pkg_resources.*"), Pattern.compile("^pkg_resources.declare_namespace\\(__name__\\).*"))) private fun isUserFile(element: PsiElement, module: Module?) = diff --git a/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py b/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py index 0bfb5a62b4f6..2f5c78e483d4 100644 --- a/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py +++ b/python/testData/resolve/multiFile/pkgutilNamespace/root2/pkg/__init__.py @@ -1,2 +1,2 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) \ No newline at end of file +import pkgutil +__path__ = pkgutil.extend_path(__path__, __name__) \ No newline at end of file diff --git a/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/__init__.py b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/__init__.py new file mode 100644 index 000000000000..a044d4ee9232 --- /dev/null +++ b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/__init__.py @@ -0,0 +1,5 @@ +# This is a comment in __init__ file +# to check that we still understand it's a part of namespace package +# + +__path__= __import__('pkgutil').extend_path(__path__, __name__) # can be on the same line \ No newline at end of file diff --git a/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/a.py b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/a.py new file mode 100644 index 000000000000..c394838ae54d --- /dev/null +++ b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root1/pkg/a.py @@ -0,0 +1,2 @@ +import pkg.second +# diff --git a/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/__init__.py b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/__init__.py new file mode 100644 index 000000000000..cddc08dbcd32 --- /dev/null +++ b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/__init__.py @@ -0,0 +1,7 @@ +# This is a comment in __init__ file + +from pkgutil import extend_path # can be on the same line + +__path__ = extend_path(__path__, __name__) + +# to check that we still understand it's a part of namespace package diff --git a/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/second.py b/python/testData/resolve/multiFile/pkgutilNamespaceWithComments/root2/pkg/second.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java b/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java index f90743abd357..1d8f42c026a7 100644 --- a/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java +++ b/python/testSrc/com/jetbrains/python/PyMultiFileResolveTest.java @@ -478,23 +478,21 @@ public class PyMultiFileResolveTest extends PyMultiFileResolveTestCase { // PY-16688 public void testPkgResourcesNamespace() { - myFixture.copyDirectoryToProject("pkgResourcesNamespace", ""); - runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> { - final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py"); - final PsiReference ref = PyResolveTestCase.findReferenceByMarker(psiFile); - assertInstanceOf(ref, PsiPolyVariantReference.class); - final List elements = PyUtil.multiResolveTopPriority((PsiPolyVariantReference)ref); - assertEquals(1, elements.size()); - PsiFile root1 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root1/pkg/__init__.py")); - PsiFile root2 = myFixture.getPsiManager().findFile(myFixture.findFileInTempDir("root2/pkg/__init__.py")); - assertNotParsed(root1); - assertNotParsed(root2); - }); + doTestResolveInNamespacePackage(getTestName(true)); } // PY-23087 public void testPkgutilNamespace() { - myFixture.copyDirectoryToProject("pkgutilNamespace", ""); + doTestResolveInNamespacePackage(getTestName(true)); + } + + // PY-38434 + public void testPkgutilNamespaceWithComments() { + doTestResolveInNamespacePackage(getTestName(true)); + } + + private void doTestResolveInNamespacePackage(String namespace) { + myFixture.copyDirectoryToProject(namespace, ""); runWithSourceRoots(Lists.newArrayList(myFixture.findFileInTempDir("root1"), myFixture.findFileInTempDir("root2")), () -> { final PsiFile psiFile = myFixture.configureByFile("root1/pkg/a.py"); final PsiReference ref = PyResolveTestCase.findReferenceByMarker(psiFile);