Consider pyi file located not in stub package or typeshed as having more priority than them (PY-32963)

It is assumed to be provided with a distribution.
As a result stub package should not replace any other package because such package could contain pyi stubs.

python/src/com/jetbrains/python/codeInsight/typing/PyStubPackages.kt

@@ -8,7 +8,6 @@ import com.intellij.openapi.module.ModuleUtilCore
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.roots.ProjectFileIndex
 import com.intellij.openapi.util.Key
-import com.intellij.openapi.util.text.StringUtil
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.PsiDirectory
 import com.intellij.psi.PsiElement
@@ -136,14 +135,7 @@ private fun getClassOrContentOrSourceRoot(project: Project, file: VirtualFile):
   return null
 }
 
-fun stubPackageIsPartial(stubPackageDirectory: PsiDirectory): Boolean {
-  val pyTyped = stubPackageDirectory.findFile("py.typed") ?: return false
-
-  return pyTyped.textLength < "partial".length + 5 &&
-         pyTyped.text.let { it.startsWith("partial") && it.substring("partial".length).all(StringUtil::isLineBreak) }
-}
-
-private fun pyi(element: PsiElement) = element is PyiFile || PyUtil.turnDirIntoInit(element) is PyiFile
+private fun isPyi(element: PsiElement) = element is PyiFile || PyUtil.turnDirIntoInit(element) is PyiFile
 
 private fun isNamespacePackage(element: PsiElement): Boolean {
   if (element is PsiDirectory) {
@@ -160,9 +152,10 @@ private fun isNamespacePackage(element: PsiElement): Boolean {
  */
 private fun resolvedElementPriority(element: PsiElement, module: Module?) = when {
   isNamespacePackage(element) -> Priority.NAMESPACE_PACKAGE
-  isUserFile(element, module) -> if (pyi(element)) Priority.USER_STUB else Priority.USER_CODE
+  isUserFile(element, module) -> if (isPyi(element)) Priority.USER_STUB else Priority.USER_CODE
   isInStubPackage(element) -> Priority.STUB_PACKAGE
   isInTypeShed(element) -> Priority.TYPESHED
+  isPyi(element) -> Priority.PROVIDED_STUB
   isInInlinePackage(element, module) -> Priority.INLINE_PACKAGE
   else -> Priority.OTHER
 }
@@ -178,7 +171,7 @@ private fun isUserFile(element: PsiElement, module: Module?) =
 fun isInStubPackage(element: PsiElement) = element.getUserData(STUB_PACKAGE_KEY) == true
 
 private fun isInTypeShed(element: PsiElement) =
-  pyi(element) && (element as? PsiFileSystemItem)?.virtualFile.let { it != null && PyTypeShed.isInside(it) }
+  isPyi(element) && (element as? PsiFileSystemItem)?.virtualFile.let { it != null && PyTypeShed.isInside(it) }
 
 /**
  * See [https://www.python.org/dev/peps/pep-0561/#packaging-type-information].
@@ -190,7 +183,7 @@ private fun isInInlinePackage(element: PsiElement, module: Module?): Boolean {
   val cached = element.getUserData(INLINE_PACKAGE_KEY)
   if (cached != null) return cached
 
-  val result = !pyi(element) && (element is PyFile || PyUtil.turnDirIntoInit(element) is PyFile) && getPyTyped(element) != null
+  val result = !isPyi(element) && (element is PyFile || PyUtil.turnDirIntoInit(element) is PyFile) && getPyTyped(element) != null
 
   element.putUserData(INLINE_PACKAGE_KEY, result)
   return result
@@ -222,5 +215,12 @@ private fun getPyTyped(element: PsiElement?): VirtualFile? {
  * Order is important, see [filterTopPriorityResults].
  */
 private enum class Priority {
-  USER_STUB, USER_CODE, STUB_PACKAGE, INLINE_PACKAGE, TYPESHED, OTHER, NAMESPACE_PACKAGE
+  USER_STUB, // pyi file located in user's project
+  USER_CODE, // py file located in user's project
+  PROVIDED_STUB, // pyi file provided with installed lib and located inside it
+  STUB_PACKAGE, // pyi file located in some stub package
+  INLINE_PACKAGE, // py file located in some inline package
+  TYPESHED, // pyi file located in typeshed
+  OTHER, // other cases, e.g. py file located inside installed lib
+  NAMESPACE_PACKAGE // namespace package has the lowest priority
 }

python/src/com/jetbrains/python/psi/resolve/ResolveImportUtil.java

@@ -382,22 +382,12 @@ public class ResolveImportUtil {
     final PsiDirectory subdir = dir.findSubdirectory(referencedName);
     // VFS may be case insensitive on Windows, but resolve is always case sensitive (PEP 235, PY-18958), so we check name here
     if (subdir != null && subdir.getName().equals(referencedName) && (!checkForPackage || PyUtil.isPackage(subdir, containingFile))) {
-      final PsiDirectory stubPackage = PyStubPackages.findStubPackage(dir, referencedName, checkForPackage, withoutStubs);
-
-      if (stubPackage == null || PyStubPackages.stubPackageIsPartial(stubPackage)) {
-        result.add(new RatedResolveResult(RatedResolveResult.RATE_NORMAL, PyStubPackages.transferStubPackageMarker(dir, subdir)));
-      }
-
-      if (stubPackage != null) {
-        result.add(new RatedResolveResult(RatedResolveResult.RATE_NORMAL, stubPackage));
-      }
+      result.add(new RatedResolveResult(RatedResolveResult.RATE_NORMAL, PyStubPackages.transferStubPackageMarker(dir, subdir)));
     }
 
-    if (subdir == null) {
-      final PsiDirectory stubPackage = PyStubPackages.findStubPackage(dir, referencedName, checkForPackage, withoutStubs);
-      if (stubPackage != null) {
-        result.add(new RatedResolveResult(RatedResolveResult.RATE_NORMAL, stubPackage));
-      }
+    final PsiDirectory stubPackage = PyStubPackages.findStubPackage(dir, referencedName, checkForPackage, withoutStubs);
+    if (stubPackage != null) {
+      result.add(new RatedResolveResult(RatedResolveResult.RATE_NORMAL, stubPackage));
     }
 
     final PsiFile module = findPyFileInDir(dir, referencedName, withoutStubs);

python/testData/resolve/ProvidedPyiInsteadStubPackage/lib/pkg/foo.pyi

@@ -0,0 +1 @@
+def bar(a: str, b: str) -> str: ...

python/testData/resolve/UserPyInsteadProvidedPyi/lib/pkg/foo.py

@@ -0,0 +1,2 @@
+def bar(a: str, b: str):
+    return a + b

python/testData/resolve/UserPyInsteadProvidedPyi/lib/pkg/foo.pyi

@@ -0,0 +1 @@
+def bar(a: str, b: str) -> str: ...

python/testData/resolve/UserPyInsteadProvidedPyi/main.py

@@ -0,0 +1,4 @@
+from pkg import foo
+
+foo.bar("a", "b")
+#    <ref>

python/testSrc/com/jetbrains/python/Py3ResolveTest.java

@@ -674,7 +674,7 @@ public class Py3ResolveTest extends PyResolveTestCase {
   }
 
   // PY-30942
-  public void testUserPyInsteadStubPackage() {
+  public void testUserPyInsteadProvidedPyi() {
     final String path = "resolve/" + getTestName(false);
     myFixture.copyDirectoryToProject(path + "/pkg", "pkg");
     myFixture.configureByFile(path + "/main.py");
@@ -695,6 +695,26 @@ public class Py3ResolveTest extends PyResolveTestCase {
     );
   }
 
+  // PY-32963
+  public void testProvidedPyiInsteadStubPackage() {
+    final String path = "resolve/" + getTestName(false);
+    myFixture.configureByFile(path + "/main.py");
+
+    final VirtualFile libDir = StandardFileSystems.local().findFileByPath(getTestDataPath() + "/" + path + "/lib");
+    assertNotNull(libDir);
+
+    runWithAdditionalClassEntryInSdkRoots(
+      libDir,
+      () -> {
+        final PsiElement element = PyResolveTestCase.findReferenceByMarker(myFixture.getFile()).resolve();
+
+        final PsiFile file = element.getContainingFile();
+        assertEquals("foo.pyi", file.getName());
+        assertEquals("pkg", file.getParent().getName());
+      }
+    );
+  }
+
   // PY-30942
   public void testStubPackageInsteadInlinePackage() {
     final String path = "resolve/" + getTestName(false);
@@ -801,8 +821,9 @@ public class Py3ResolveTest extends PyResolveTestCase {
     );
   }
 
+  // TODO: this should be fixed after introducing an ability to check visited paths while resolving some qualified name
   // PY-30942
-  public void testNoInlinePackageInsteadStubPackage() {
+  public void _testNoInlinePackageInsteadStubPackage() {
     final String path = "resolve/" + getTestName(false);
     myFixture.configureByFile(path + "/main.py");
 
@@ -815,8 +836,9 @@ public class Py3ResolveTest extends PyResolveTestCase {
     );
   }
 
+  // TODO: this should be fixed after introducing an ability to check visited paths while resolving some qualified name
   // PY-30942
-  public void testNoInlinePackageInsteadStubPackageAnotherImport() {
+  public void _testNoInlinePackageInsteadStubPackageAnotherImport() {
     final String path = "resolve/" + getTestName(false);
     myFixture.configureByFile(path + "/main.py");