PY-75831 Split cache(resolve/type) into library and user part

(cherry picked from commit 94456d3a1f0be2e747641b74ce8246857b1a3f77)

IJ-MR-169158

GitOrigin-RevId: 4cdf0b90f601ca37b0f11b4392dbcfcf856472c6

python/python-psi-api/BUILD.bazel

@@ -23,6 +23,7 @@ jvm_library(
     "//python/python-ast:ast",
     "//python/python-syntax-core:syntax-core",
     "@lib//:kotlin-stdlib",
+    "//platform/platform-api:ide",
   ],
   exports = [
     "//python/python-parser:parser",

python/python-psi-api/intellij.python.psi.iml

@@ -20,5 +20,6 @@
     <orderEntry type="module" module-name="intellij.python.syntax.core" />
     <orderEntry type="library" name="kotlin-stdlib" level="project" />
     <orderEntry type="module" module-name="intellij.platform.backend" scope="RUNTIME" />
+    <orderEntry type="module" module-name="intellij.platform.ide" />
   </component>
 </module>

python/python-psi-api/src/com/jetbrains/python/psi/types/PyLibraryModificationTracker.kt

@@ -0,0 +1,80 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.psi.types
+
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.components.Service
+import com.intellij.openapi.components.service
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.diagnostic.ThrottledLogger
+import com.intellij.openapi.fileEditor.FileDocumentManagerListener
+import com.intellij.openapi.project.DumbService
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.ProjectRootManager
+import com.intellij.openapi.util.ModificationTracker
+import com.intellij.openapi.util.SimpleModificationTracker
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.PsiManager
+import com.intellij.psi.PsiTreeChangeAdapter
+import com.intellij.psi.PsiTreeChangeEvent
+import com.intellij.psi.search.GlobalSearchScope
+import com.intellij.psi.search.ProjectScope
+import com.intellij.util.ForcefulReparseModificationTracker
+import java.util.concurrent.TimeUnit
+
+@Service(Service.Level.PROJECT)
+class PyLibraryModificationTracker(project: Project) : ModificationTracker, Disposable {
+  private val myProjectRootManager: ModificationTracker = ProjectRootManager.getInstance(project)
+  private val myDumbServiceModificationTracker: ModificationTracker = DumbService.getInstance(project).modificationTracker
+  private val myForcefulReparseModificationTracker: ModificationTracker = ForcefulReparseModificationTracker.getInstance() // PsiClass from libraries may become invalid on reparse
+  private val myOnContentReloadModificationTracker: SimpleModificationTracker = SimpleModificationTracker()
+  private val creationStack = Throwable()
+  val projectLibraryScope: GlobalSearchScope = ProjectScope.getLibrariesScope(project)
+
+  init {
+    val connection = project.getMessageBus().connect(this)
+    PsiManager.getInstance(project).addPsiTreeChangeListener(object : PsiTreeChangeAdapter() {
+      override fun childrenChanged(event: PsiTreeChangeEvent) {
+        val file = event.file ?: return
+        val virtualFile = file.virtualFile ?: return
+        if (isLibraryFile(virtualFile)) {
+          myOnContentReloadModificationTracker.incModificationCount()
+        }
+      }
+    }, this)
+
+    connection.subscribe<FileDocumentManagerListener>(FileDocumentManagerListener.TOPIC, object : FileDocumentManagerListener {
+      override fun fileWithNoDocumentChanged(file: VirtualFile) {
+        if (!project.isInitialized()) {
+          THROTTLED_LOG.warn("SearchScope.contains(file) would log an error because WorkspaceFileIndex is not yet initialized. " +
+                             "Probably LibraryModificationTracker was created too early. " +
+                             "See LibraryModificationTracker creation stacktrace: ", creationStack)
+          return
+        }
+        if (isLibraryFile(file)) {
+          myOnContentReloadModificationTracker.incModificationCount()
+        }
+      }
+    })
+  }
+
+  private fun isLibraryFile(file: VirtualFile): Boolean {
+    return "pyi" == file.extension || projectLibraryScope.contains(file)
+  }
+
+  override fun getModificationCount(): Long {
+    return (myProjectRootManager.getModificationCount()
+            + myDumbServiceModificationTracker.getModificationCount()
+            + myForcefulReparseModificationTracker.getModificationCount()
+            + myOnContentReloadModificationTracker.getModificationCount())
+  }
+
+
+  override fun dispose() {
+  }
+
+  companion object {
+    private val THROTTLED_LOG = ThrottledLogger(Logger.getInstance(PyLibraryModificationTracker::class.java), TimeUnit.SECONDS.toMillis(30))
+
+    fun getInstance(project: Project): PyLibraryModificationTracker = project.service()
+  }
+}

python/python-psi-api/src/com/jetbrains/python/psi/types/TypeEvalContext.java

@@ -2,10 +2,12 @@
 package com.jetbrains.python.psi.types;
 
 import com.intellij.openapi.project.Project;
+import com.intellij.openapi.roots.ProjectFileIndex;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.RecursionManager;
 import com.intellij.openapi.util.registry.Registry;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
 import com.intellij.util.ArrayUtil;
@@ -97,6 +99,9 @@ public sealed class TypeEvalContext {
    * the analyzed code was called or may be called. Since this is basically guesswork, the results should be used only for code completion.
    */
   public static @NotNull TypeEvalContext codeCompletion(final @NotNull Project project, final @Nullable PsiFile origin) {
+    if (Registry.is("python.use.library.leve.type.eval.context")) {
+      return new LibraryLongLiveTypeEvalContext(true, true, true, origin);
+    }
     return getContextFromCache(project, new TypeEvalContext(true, true, true, origin));
   }
 
@@ -108,6 +113,9 @@ public sealed class TypeEvalContext {
    * For code completion see {@link TypeEvalContext#codeCompletion(Project, PsiFile)}.
    */
   public static TypeEvalContext userInitiated(final @NotNull Project project, final @Nullable PsiFile origin) {
+    if (Registry.is("python.use.library.leve.type.eval.context")) {
+      return new LibraryLongLiveTypeEvalContext(true, true, false, origin);
+    }
     return getContextFromCache(project, new TypeEvalContext(true, true, false, origin));
   }
 
@@ -424,6 +432,60 @@ public sealed class TypeEvalContext {
     }
   }
 
+  final static class LibraryLongLiveTypeEvalContext extends TypeEvalContext {
+    private LibraryLongLiveTypeEvalContext(boolean allowDataFlow,
+                                           boolean allowStubToAST,
+                                           boolean allowCallContext,
+                                           @Nullable PsiFile origin) {
+      super(allowDataFlow, allowStubToAST, allowCallContext, origin);
+    }
+
+    private static boolean isInLibrary(@NotNull PsiElement element) {
+      VirtualFile vFile = element.getContainingFile().getOriginalFile().getVirtualFile();
+      return vFile != null && ("pyi".equals(vFile.getExtension()) || ProjectFileIndex.getInstance(element.getProject()).isInLibrary(vFile));
+    }
+
+    private @NotNull TypeEvalContext getLibraryContext(@NotNull Project project) {
+      return project.getService(TypeEvalContextCache.class).getLibraryContext(new TypeEvalContext(getConstraints()));
+    }
+
+    @Override
+    protected @Nullable PyType getKnownType(@NotNull PyTypedElement element) {
+      if (!isInLibrary(element)) {
+        return super.getKnownType(element);
+      }
+      var context = getLibraryContext(element.getProject());
+      return context.getKnownType(element);
+    }
+
+    @Override
+    protected @Nullable PyType getKnownReturnType(@NotNull PyCallable callable) {
+      if (!isInLibrary(callable)) {
+        return super.getKnownReturnType(callable);
+      }
+      var context = getLibraryContext(callable.getProject());
+      return context.getKnownReturnType(callable);
+    }
+
+    @Override
+    public @Nullable PyType getType(@NotNull PyTypedElement element) {
+      if (!isInLibrary(element)) {
+        return super.getType(element);
+      }
+      var context = getLibraryContext(element.getProject());
+      return context.getType(element);
+    }
+
+    @Override
+    public @Nullable PyType getReturnType(@NotNull PyCallable callable) {
+      if (!isInLibrary(callable)) {
+        return super.getReturnType(callable);
+      }
+      var context = getLibraryContext(callable.getProject());
+      return context.getReturnType(callable);
+    }
+  }
+
   final static class OptimizedTypeEvalContext extends TypeEvalContext {
     private volatile TypeEvalContext codeInsightFallback;
 

python/python-psi-api/src/com/jetbrains/python/psi/types/TypeEvalContextCache.java

@@ -36,4 +36,7 @@ public interface TypeEvalContextCache {
    */
   @NotNull
   TypeEvalContext getContext(@NotNull TypeEvalContext standard);
+
+  @NotNull
+  TypeEvalContext getLibraryContext(@NotNull TypeEvalContext standard);
 }