IJPL-200056: Generate library iterators from WorkspaceFileIndexChanged events

GitOrigin-RevId: 8f0f621a47168ef5635ae5e01c644f49505cb4cd
2026-10-06 06:21:16 +07:00 · 2025-08-18 14:09:19 +00:00
parent 405b528027
commit 62b5c31317
10 changed files with 178 additions and 13 deletions
@@ -4,6 +4,7 @@ package com.intellij.util.indexing
 import com.intellij.ide.lightEdit.LightEdit
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.ModalityState
+import com.intellij.openapi.application.ReadAction
 import com.intellij.openapi.components.Service
 import com.intellij.openapi.components.service
 import com.intellij.openapi.components.serviceAsync
@@ -26,20 +27,26 @@ import com.intellij.util.indexing.EntityIndexingServiceImpl.WorkspaceEntitiesRoo
 import com.intellij.util.indexing.EntityIndexingServiceImpl.WorkspaceEventRescanningInfo
 import com.intellij.util.indexing.dependenciesCache.DependenciesIndexedStatusService
 import com.intellij.util.indexing.dependenciesCache.DependenciesIndexedStatusService.StatusMark
+import com.intellij.util.indexing.roots.GenericDependencyIterator
 import com.intellij.util.indexing.roots.IndexableEntityProvider
 import com.intellij.util.indexing.roots.IndexableEntityProvider.*
 import com.intellij.util.indexing.roots.IndexableFilesIterator
 import com.intellij.util.indexing.roots.WorkspaceIndexingRootsBuilder
 import com.intellij.util.indexing.roots.builders.IndexableIteratorBuilders
 import com.intellij.util.indexing.roots.builders.IndexableIteratorBuilders.forLibraryEntity
+import com.intellij.util.indexing.roots.kind.LibraryOrigin
+import com.intellij.util.indexing.roots.origin.LibraryOriginImpl
 import com.intellij.workspaceModel.core.fileIndex.*
 import com.intellij.workspaceModel.core.fileIndex.DependencyDescription.OnParent
+import com.intellij.workspaceModel.core.fileIndex.impl.ModuleRelatedRootData
 import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexImpl.Companion.EP_NAME
+import com.intellij.workspaceModel.core.fileIndex.impl.getEntityPointer
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Deferred
 import kotlinx.coroutines.async
 import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.annotations.TestOnly
+import java.util.concurrent.Callable


@ApiStatus.Internal
@@ -48,7 +55,7 @@ import org.jetbrains.annotations.TestOnly
 class ProjectEntityIndexingService(
  private val project: Project,
  private val scope: CoroutineScope,
-): WorkspaceFileIndexListener {
+) : WorkspaceFileIndexListener {

  private val tracker = CustomEntitiesCausingReindexTracker()

@@ -72,6 +79,41 @@ class ProjectEntityIndexingService(
  }

  override fun workspaceFileIndexChanged(event: WorkspaceFileIndexChangedEvent) {
+    if (!Registry.`is`("use.workspace.file.index.for.partial.scanning")) return
+    if (FileBasedIndex.getInstance() !is FileBasedIndexImpl) return
+    if (LightEdit.owns(project)) return
+    if (invalidateProjectFilterIfFirstScanningNotRequested(project)) return
+
+    if (ModalityState.defaultModalityState() === ModalityState.any()) {
+      LOG.error("Unexpected modality: should not be ANY. Replace with NON_MODAL (130820241337)")
+    }
+
+    if (event.registeredFileSets.isNotEmpty() || event.removedFileSets.isNotEmpty()) {
+      val parameters =  computeScanningParametersFromWFIEvent(event)
+      UnindexedFilesScanner(project, parameters).queue()
+    }
+  }
+
+  private fun computeScanningParametersFromWFIEvent(event: WorkspaceFileIndexChangedEvent): Deferred<ScanningParameters> {
+    return scope.async {
+      ReadAction.nonBlocking(Callable {
+        val iterators = ArrayList<IndexableFilesIterator>()
+
+        //for (fileSets in event.removedFileSets) {
+        //  generateIteratorsFromWFIChangedEvent(fileSets, event.storageBefore, iterators)
+        //}
+        for (fileSets in event.registeredFileSets) {
+          generateIteratorsFromWFIChangedEvent(fileSets, event.storageAfter, iterators)
+        }
+
+        return@Callable if (iterators.isEmpty()) {
+          CancelledScanning
+        }
+        else {
+          ScanningIterators("Changes from WorkspaceFileIndex", predefinedIndexableFilesIterators = iterators)
+        }
+      }).executeSynchronously()
+    }
  }

  private enum class Change {
@@ -91,6 +133,37 @@ class ProjectEntityIndexingService(
    return tracker.shouldRescan(oldEntity, newEntity, project)
  }

+  private fun generateIteratorsFromWFIChangedEvent(
+    fileSets: Collection<WorkspaceFileSet>,
+    storage: EntityStorage,
+    iterators: MutableList<IndexableFilesIterator>,
+  ) {
+    val libraryOrigins = HashSet<LibraryOrigin>()
+
+    for (fileSet in fileSets) {
+      fileSet as WorkspaceFileSetWithCustomData<*>
+      val entityPointer = fileSet.getEntityPointer() ?: continue
+      if (!fileSet.kind.isIndexable) continue
+      if (fileSet.data is ModuleRelatedRootData) continue
+      if (fileSet.kind.isContent) continue
+
+      val entity = entityPointer.resolve(storage) ?: continue
+      if (entity is LibraryEntity) {
+        val sourceRoot = fileSet.kind == WorkspaceFileKind.EXTERNAL_SOURCE
+        val origin = if (sourceRoot) {
+          LibraryOriginImpl(emptyList(), listOf(fileSet.root))
+        }
+        else {
+          LibraryOriginImpl(listOf(fileSet.root), emptyList())
+        }
+        val iterator = GenericDependencyIterator.forLibraryEntity(origin, entity.name, fileSet.root, sourceRoot)
+        if (libraryOrigins.add(origin)) {
+          iterators.add(iterator)
+        }
+      }
+    }
+  }
+
  private fun computeScanningParameters(changes: List<RootsChangeRescanningInfo>): Deferred<ScanningParameters> {
    return scope.async {
      var indexDependencies = false
@@ -0,0 +1,61 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.util.indexing.roots
+
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.ContentIterator
+import com.intellij.openapi.util.NlsContexts
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileFilter
+import com.intellij.util.indexing.IndexingBundle
+import com.intellij.util.indexing.roots.kind.IndexableSetOrigin
+import org.jetbrains.annotations.ApiStatus
+import org.jetbrains.annotations.NonNls
+import org.jetbrains.annotations.Unmodifiable
+
+@ApiStatus.Internal
+class GenericDependencyIterator(
+  private val origin: IndexableSetOrigin,
+  private val root: VirtualFile,
+  private val indexingProgressText: @NlsContexts.ProgressText String,
+  private val rootsScanningProgressText: @NlsContexts.ProgressText String,
+  private val debugName: String,
+) : IndexableFilesIterator {
+  override fun getDebugName(): @NonNls String {
+    return debugName
+  }
+
+  override fun getIndexingProgressText(): @NlsContexts.ProgressText String {
+    return indexingProgressText
+  }
+
+  override fun getRootsScanningProgressText(): @NlsContexts.ProgressText String {
+    return rootsScanningProgressText
+  }
+
+  override fun getOrigin(): IndexableSetOrigin {
+    return origin
+  }
+
+  override fun iterateFiles(project: Project, fileIterator: ContentIterator, fileFilter: VirtualFileFilter): Boolean {
+    return IndexableFilesIterationMethods.iterateRoots(project, listOf(root), fileIterator, fileFilter)
+  }
+
+  override fun getRootUrls(project: Project): @Unmodifiable Set<String> {
+    return setOf(root.url)
+  }
+
+  companion object {
+    fun forLibraryEntity(origin: IndexableSetOrigin, libraryName: String, root: VirtualFile, sourceRoot: Boolean): IndexableFilesIterator {
+      val debugMessage = if (sourceRoot) {
+        "(source root ${root.name})"
+      }
+      else {
+        "(class root ${root.name})"
+      }
+      return GenericDependencyIterator(origin, root,
+                                       indexingProgressText = IndexingBundle.message("indexable.files.provider.indexing.library.name", libraryName),
+                                       rootsScanningProgressText = IndexingBundle.message("indexable.files.provider.scanning.library.name", libraryName),
+                                       debugName = "Library ${libraryName} $debugMessage")
+    }
+  }
+}
@@ -11,6 +11,7 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.Condition
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.OSAgnosticPathUtil
+import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.platform.backend.workspace.virtualFile
 import com.intellij.platform.workspace.jps.entities.LibraryEntity
@@ -143,10 +144,6 @@ internal fun selectRootVirtualFiles(value: Collection<VirtualFile>): List<Virtua
  return selectRootItems(value) { file -> file.path }
 }

-internal fun selectRootVirtualFileUrls(urls: Collection<VirtualFileUrl>): List<VirtualFileUrl> {
-  return selectRootItems(urls) { url -> JpsPathUtil.urlToPath(url.url) }
-}
-
 private fun <T> selectRootItems(items: Collection<T>, toPath: Function<T, String>): List<T> {
  if (items.size < 2) {
    if (items is List<T>) return items
@@ -261,11 +258,15 @@ internal class WorkspaceIndexingRootsBuilder(private val ignoreModuleRoots: Bool
    }

    for ((libraryEntity, roots) in rootData.libraryRoots.entries) {
-      descriptions.add(LibraryRootsDescription(libraryEntity, roots))
+      if (!Registry.`is`("use.workspace.file.index.for.partial.scanning")) {
+        descriptions.add(LibraryRootsDescription(libraryEntity, roots))
+      }
    }

    for ((libraryEntity, roots) in rootData.libraryUrlRoots.entries) {
-      descriptions.add(LibraryUrlRootsDescription(libraryEntity, roots))
+      if (!Registry.`is`("use.workspace.file.index.for.partial.scanning")) {
+        descriptions.add(LibraryUrlRootsDescription(libraryEntity, roots))
+      }
    }

    for ((entityReference, roots) in rootData.externalRoots.entries) {
@@ -2,6 +2,7 @@
 package com.intellij.util.indexing.roots;

 import com.intellij.openapi.project.Project;
+import com.intellij.openapi.util.registry.Registry;
 import com.intellij.platform.workspace.jps.entities.*;
 import com.intellij.util.SmartList;
 import com.intellij.util.indexing.roots.builders.IndexableIteratorBuilders;
@@ -56,6 +57,7 @@ public final class ModuleDependencyEntitiesIndexableEntityProvider implements In
                                                                                 ((SdkDependency)dependency).getSdk().getType()));
    }
    else if (dependency instanceof LibraryDependency) {
+      if (Registry.is("use.workspace.file.index.for.partial.scanning")) return Collections.emptyList();
      LibraryId libraryId = ((LibraryDependency)dependency).getLibrary();
      return IndexableIteratorBuilders.INSTANCE.forLibraryEntity(libraryId, true);
    }
@@ -3,6 +3,7 @@ package com.intellij.util.indexing.roots.builders

 import com.intellij.openapi.application.ReadAction
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.platform.workspace.jps.entities.*
 import com.intellij.platform.workspace.storage.EntityStorage
@@ -16,6 +17,7 @@ import com.intellij.workspaceModel.ide.legacyBridge.ModuleDependencyIndex

 class LibraryIndexableIteratorHandler : IndexableIteratorBuilderHandler {
  override fun accepts(builder: IndexableEntityProvider.IndexableIteratorBuilder): Boolean =
+    !Registry.`is`("use.workspace.file.index.for.partial.scanning") &&
    builder is LibraryIdIteratorBuilder

  override fun instantiate(builders: Collection<IndexableEntityProvider.IndexableIteratorBuilder>,
@@ -2,7 +2,6 @@
 package com.intellij.util.indexing.roots.origin

 import com.intellij.openapi.module.Module
-import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.roots.SyntheticLibrary
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.platform.workspace.storage.EntityPointer
@@ -8,6 +8,7 @@ import com.intellij.openapi.roots.impl.OrderRootsCache
 import com.intellij.openapi.roots.impl.ProjectRootManagerComponent
 import com.intellij.openapi.roots.libraries.Library
 import com.intellij.openapi.util.EmptyRunnable
+import com.intellij.openapi.util.registry.Registry
 import com.intellij.platform.workspace.jps.entities.LibraryTableId
 import com.intellij.util.indexing.BuildableRootsChangeRescanningInfo
 import com.intellij.workspaceModel.ide.impl.legacyBridge.library.LibraryBridge
@@ -55,12 +56,14 @@ class ProjectRootManagerBridge(project: Project, coroutineScope: CoroutineScope)
    private var insideRootsChange = false

    override fun referencedLibraryAdded(library: Library) {
+      if (Registry.`is`("use.workspace.file.index.for.partial.scanning")) return
      if (shouldListen(library)) {
        fireRootsChanged(BuildableRootsChangeRescanningInfo.newInstance().addLibrary(library).buildInfo())
      }
    }

    override fun referencedLibraryChanged(library: Library) {
+      if (Registry.`is`("use.workspace.file.index.for.partial.scanning")) return
      if (insideRootsChange || !shouldListen(library)) return
      insideRootsChange = true
      try {
@@ -12,6 +12,7 @@ import com.intellij.openapi.roots.libraries.Library;
 import com.intellij.openapi.roots.libraries.LibraryTable;
 import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar;
 import com.intellij.openapi.roots.ui.configuration.SdkTestCase;
+import com.intellij.openapi.util.registry.Registry;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.platform.workspace.jps.entities.LibraryId;
 import com.intellij.platform.workspace.jps.entities.LibraryTableId;
@@ -50,7 +51,9 @@ public class EntityIndexingServiceTest extends EntityIndexingServiceTestBase {
  }

  public void testIndexingProjectLibrary() throws Exception {
-    doTest(this::createProjectLibrary, this::removeProjectLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList);
+    if (!Registry.is("use.workspace.file.index.for.partial.scanning", false)) {
+      doTest(this::createProjectLibrary, this::removeProjectLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList);
+    }
  }

  @NotNull
@@ -84,8 +87,10 @@ public class EntityIndexingServiceTest extends EntityIndexingServiceTestBase {
  }

  public void testIndexingGlobalLibrary() throws Exception {
-    doTest(this::createGlobalLibrary, this::removeGlobalLibrary,
-           pair -> LibraryIndexableFilesIteratorImpl.createIteratorList(pair.getFirst()));
+    if (!Registry.is("use.workspace.file.index.for.partial.scanning", false)) {
+      doTest(this::createGlobalLibrary, this::removeGlobalLibrary,
+             pair -> LibraryIndexableFilesIteratorImpl.createIteratorList(pair.getFirst()));
+    }
  }

  @NotNull
@@ -100,7 +105,9 @@ public class EntityIndexingServiceTest extends EntityIndexingServiceTestBase {
  }

  public void testIndexingModuleLibrary() throws Exception {
-    doTest(this::createModuleLibrary, this::removeModuleLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList);
+    if (!Registry.is("use.workspace.file.index.for.partial.scanning", false)) {
+      doTest(this::createModuleLibrary, this::removeModuleLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList);
+    }
  }

  private void removeModuleLibrary(Library library) {
@@ -47,6 +47,8 @@
                 description="Removes custom processing of Global Libraries and SDKs in the Workspace Model"/>
    <registryKey key="ide.workspace.model.per.environment.model.separation" defaultValue="false" restartRequired="true"
                 description="Enable existence of multiple workspace models, one per each isolated environment"/>
+    <registryKey key="use.workspace.file.index.for.partial.scanning" defaultValue="true" restartRequired="true"
+                 description="If enabled, relies on events from WorkspaceFileIndex for partial scanning"/>

    <persistentFsConnectionListener implementation="com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexCleaner"/>
  </extensions>
@@ -10,11 +10,13 @@ import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileVisitor
 import com.intellij.platform.backend.workspace.virtualFile
+import com.intellij.platform.workspace.jps.entities.LibraryDependency
 import com.intellij.platform.workspace.jps.entities.LibraryEntity
 import com.intellij.platform.workspace.jps.entities.LibraryId
 import com.intellij.platform.workspace.jps.entities.LibraryRoot.InclusionOptions.*
 import com.intellij.platform.workspace.jps.entities.LibraryRootTypeId
 import com.intellij.platform.workspace.jps.entities.LibraryTableId
+import com.intellij.platform.workspace.jps.entities.ModuleEntity
 import com.intellij.platform.workspace.storage.EntityStorage
 import com.intellij.platform.workspace.storage.url.VirtualFileUrl
 import com.intellij.util.asSafely
@@ -32,6 +34,12 @@ class LibraryRootFileIndexContributor : WorkspaceFileIndexContributor<LibraryEnt
      if (entity.symbolicId.tableId is LibraryTableId.GlobalLibraryTableId) return
      entity.symbolicId.takeIf { it.tableId == LibraryTableId.ProjectLibraryTableId }
    }
+    if (Registry.`is`("ide.workspace.model.sdk.remove.custom.processing") && libraryId != null) {
+      if (libraryId.tableId !is LibraryTableId.ModuleLibraryTableId &&
+          storage.referrers(libraryId, ModuleEntity::class.java).none()) {
+        return
+      }
+    }
    val compiledRootsData = LibraryRootFileSetData(libraryId)
    val sourceRootFileSetData = LibrarySourceRootFileSetData(libraryId)
    for (root in entity.roots) {
@@ -45,7 +53,7 @@ class LibraryRootFileIndexContributor : WorkspaceFileIndexContributor<LibraryEnt
        LibraryRootTypeId.SOURCES -> {
          data = sourceRootFileSetData
          kind = WorkspaceFileKind.EXTERNAL_SOURCE
-        } 
+        }
        else -> continue
      }
      when (root.inclusionOptions) {
@@ -56,6 +64,13 @@ class LibraryRootFileIndexContributor : WorkspaceFileIndexContributor<LibraryEnt
    }
  }

+  override val dependenciesOnOtherEntities: List<DependencyDescription<LibraryEntity>>
+    get() = listOf(
+      DependencyDescription.OnReference(ModuleEntity::class.java) { moduleEntity ->
+        moduleEntity.dependencies.asSequence().filterIsInstance<LibraryDependency>().map { it.library }
+      }
+    )
+
  private fun registerArchivesUnderRoot(root: VirtualFileUrl,
                                        registrar: WorkspaceFileSetRegistrar,
                                        data: LibraryRootFileSetData,