From 62b5c31317ed5b0207a8e2782e0bb2042fdb6fac Mon Sep 17 00:00:00 2001 From: Ilya Korennoy Date: Fri, 1 Aug 2025 17:04:03 +0200 Subject: [PATCH] IJPL-200056: Generate library iterators from WorkspaceFileIndexChanged events GitOrigin-RevId: 8f0f621a47168ef5635ae5e01c644f49505cb4cd --- .../indexing/ProjectEntityIndexingService.kt | 75 ++++++++++++++++++- .../roots/GenericDependencyIterator.kt | 61 +++++++++++++++ .../roots/IndexingRootsCollectionUtil.kt | 13 ++-- ...ndencyEntitiesIndexableEntityProvider.java | 2 + .../LibraryIndexableIteratorHandler.kt | 2 + .../roots/origin/indexableSetOrigins.kt | 1 - .../project/ProjectRootManagerBridge.kt | 3 + .../indexing/EntityIndexingServiceTest.java | 15 +++- .../resources/META-INF/WorkspaceModelImpl.xml | 2 + .../fileIndex/impl/libraryRootContributors.kt | 17 ++++- 10 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 platform/lang-impl/src/com/intellij/util/indexing/roots/GenericDependencyIterator.kt diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ProjectEntityIndexingService.kt b/platform/lang-impl/src/com/intellij/util/indexing/ProjectEntityIndexingService.kt index 4a6ddcb3e5fc..200a78a98287 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ProjectEntityIndexingService.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/ProjectEntityIndexingService.kt @@ -4,6 +4,7 @@ package com.intellij.util.indexing import com.intellij.ide.lightEdit.LightEdit import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.ModalityState +import com.intellij.openapi.application.ReadAction import com.intellij.openapi.components.Service import com.intellij.openapi.components.service import com.intellij.openapi.components.serviceAsync @@ -26,20 +27,26 @@ import com.intellij.util.indexing.EntityIndexingServiceImpl.WorkspaceEntitiesRoo import com.intellij.util.indexing.EntityIndexingServiceImpl.WorkspaceEventRescanningInfo import com.intellij.util.indexing.dependenciesCache.DependenciesIndexedStatusService import com.intellij.util.indexing.dependenciesCache.DependenciesIndexedStatusService.StatusMark +import com.intellij.util.indexing.roots.GenericDependencyIterator import com.intellij.util.indexing.roots.IndexableEntityProvider import com.intellij.util.indexing.roots.IndexableEntityProvider.* import com.intellij.util.indexing.roots.IndexableFilesIterator import com.intellij.util.indexing.roots.WorkspaceIndexingRootsBuilder import com.intellij.util.indexing.roots.builders.IndexableIteratorBuilders import com.intellij.util.indexing.roots.builders.IndexableIteratorBuilders.forLibraryEntity +import com.intellij.util.indexing.roots.kind.LibraryOrigin +import com.intellij.util.indexing.roots.origin.LibraryOriginImpl import com.intellij.workspaceModel.core.fileIndex.* import com.intellij.workspaceModel.core.fileIndex.DependencyDescription.OnParent +import com.intellij.workspaceModel.core.fileIndex.impl.ModuleRelatedRootData import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexImpl.Companion.EP_NAME +import com.intellij.workspaceModel.core.fileIndex.impl.getEntityPointer import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Deferred import kotlinx.coroutines.async import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.TestOnly +import java.util.concurrent.Callable @ApiStatus.Internal @@ -48,7 +55,7 @@ import org.jetbrains.annotations.TestOnly class ProjectEntityIndexingService( private val project: Project, private val scope: CoroutineScope, -): WorkspaceFileIndexListener { +) : WorkspaceFileIndexListener { private val tracker = CustomEntitiesCausingReindexTracker() @@ -72,6 +79,41 @@ class ProjectEntityIndexingService( } override fun workspaceFileIndexChanged(event: WorkspaceFileIndexChangedEvent) { + if (!Registry.`is`("use.workspace.file.index.for.partial.scanning")) return + if (FileBasedIndex.getInstance() !is FileBasedIndexImpl) return + if (LightEdit.owns(project)) return + if (invalidateProjectFilterIfFirstScanningNotRequested(project)) return + + if (ModalityState.defaultModalityState() === ModalityState.any()) { + LOG.error("Unexpected modality: should not be ANY. Replace with NON_MODAL (130820241337)") + } + + if (event.registeredFileSets.isNotEmpty() || event.removedFileSets.isNotEmpty()) { + val parameters = computeScanningParametersFromWFIEvent(event) + UnindexedFilesScanner(project, parameters).queue() + } + } + + private fun computeScanningParametersFromWFIEvent(event: WorkspaceFileIndexChangedEvent): Deferred { + return scope.async { + ReadAction.nonBlocking(Callable { + val iterators = ArrayList() + + //for (fileSets in event.removedFileSets) { + // generateIteratorsFromWFIChangedEvent(fileSets, event.storageBefore, iterators) + //} + for (fileSets in event.registeredFileSets) { + generateIteratorsFromWFIChangedEvent(fileSets, event.storageAfter, iterators) + } + + return@Callable if (iterators.isEmpty()) { + CancelledScanning + } + else { + ScanningIterators("Changes from WorkspaceFileIndex", predefinedIndexableFilesIterators = iterators) + } + }).executeSynchronously() + } } private enum class Change { @@ -91,6 +133,37 @@ class ProjectEntityIndexingService( return tracker.shouldRescan(oldEntity, newEntity, project) } + private fun generateIteratorsFromWFIChangedEvent( + fileSets: Collection, + storage: EntityStorage, + iterators: MutableList, + ) { + val libraryOrigins = HashSet() + + for (fileSet in fileSets) { + fileSet as WorkspaceFileSetWithCustomData<*> + val entityPointer = fileSet.getEntityPointer() ?: continue + if (!fileSet.kind.isIndexable) continue + if (fileSet.data is ModuleRelatedRootData) continue + if (fileSet.kind.isContent) continue + + val entity = entityPointer.resolve(storage) ?: continue + if (entity is LibraryEntity) { + val sourceRoot = fileSet.kind == WorkspaceFileKind.EXTERNAL_SOURCE + val origin = if (sourceRoot) { + LibraryOriginImpl(emptyList(), listOf(fileSet.root)) + } + else { + LibraryOriginImpl(listOf(fileSet.root), emptyList()) + } + val iterator = GenericDependencyIterator.forLibraryEntity(origin, entity.name, fileSet.root, sourceRoot) + if (libraryOrigins.add(origin)) { + iterators.add(iterator) + } + } + } + } + private fun computeScanningParameters(changes: List): Deferred { return scope.async { var indexDependencies = false diff --git a/platform/lang-impl/src/com/intellij/util/indexing/roots/GenericDependencyIterator.kt b/platform/lang-impl/src/com/intellij/util/indexing/roots/GenericDependencyIterator.kt new file mode 100644 index 000000000000..bd5142a967b1 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/roots/GenericDependencyIterator.kt @@ -0,0 +1,61 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.util.indexing.roots + +import com.intellij.openapi.project.Project +import com.intellij.openapi.roots.ContentIterator +import com.intellij.openapi.util.NlsContexts +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.VirtualFileFilter +import com.intellij.util.indexing.IndexingBundle +import com.intellij.util.indexing.roots.kind.IndexableSetOrigin +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.NonNls +import org.jetbrains.annotations.Unmodifiable + +@ApiStatus.Internal +class GenericDependencyIterator( + private val origin: IndexableSetOrigin, + private val root: VirtualFile, + private val indexingProgressText: @NlsContexts.ProgressText String, + private val rootsScanningProgressText: @NlsContexts.ProgressText String, + private val debugName: String, +) : IndexableFilesIterator { + override fun getDebugName(): @NonNls String { + return debugName + } + + override fun getIndexingProgressText(): @NlsContexts.ProgressText String { + return indexingProgressText + } + + override fun getRootsScanningProgressText(): @NlsContexts.ProgressText String { + return rootsScanningProgressText + } + + override fun getOrigin(): IndexableSetOrigin { + return origin + } + + override fun iterateFiles(project: Project, fileIterator: ContentIterator, fileFilter: VirtualFileFilter): Boolean { + return IndexableFilesIterationMethods.iterateRoots(project, listOf(root), fileIterator, fileFilter) + } + + override fun getRootUrls(project: Project): @Unmodifiable Set { + return setOf(root.url) + } + + companion object { + fun forLibraryEntity(origin: IndexableSetOrigin, libraryName: String, root: VirtualFile, sourceRoot: Boolean): IndexableFilesIterator { + val debugMessage = if (sourceRoot) { + "(source root ${root.name})" + } + else { + "(class root ${root.name})" + } + return GenericDependencyIterator(origin, root, + indexingProgressText = IndexingBundle.message("indexable.files.provider.indexing.library.name", libraryName), + rootsScanningProgressText = IndexingBundle.message("indexable.files.provider.scanning.library.name", libraryName), + debugName = "Library ${libraryName} $debugMessage") + } + } +} \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/roots/IndexingRootsCollectionUtil.kt b/platform/lang-impl/src/com/intellij/util/indexing/roots/IndexingRootsCollectionUtil.kt index 87fe6df03207..78a2c067783e 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/roots/IndexingRootsCollectionUtil.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/roots/IndexingRootsCollectionUtil.kt @@ -11,6 +11,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.util.Condition import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.io.OSAgnosticPathUtil +import com.intellij.openapi.util.registry.Registry import com.intellij.openapi.vfs.VirtualFile import com.intellij.platform.backend.workspace.virtualFile import com.intellij.platform.workspace.jps.entities.LibraryEntity @@ -143,10 +144,6 @@ internal fun selectRootVirtualFiles(value: Collection): List file.path } } -internal fun selectRootVirtualFileUrls(urls: Collection): List { - return selectRootItems(urls) { url -> JpsPathUtil.urlToPath(url.url) } -} - private fun selectRootItems(items: Collection, toPath: Function): List { if (items.size < 2) { if (items is List) return items @@ -261,11 +258,15 @@ internal class WorkspaceIndexingRootsBuilder(private val ignoreModuleRoots: Bool } for ((libraryEntity, roots) in rootData.libraryRoots.entries) { - descriptions.add(LibraryRootsDescription(libraryEntity, roots)) + if (!Registry.`is`("use.workspace.file.index.for.partial.scanning")) { + descriptions.add(LibraryRootsDescription(libraryEntity, roots)) + } } for ((libraryEntity, roots) in rootData.libraryUrlRoots.entries) { - descriptions.add(LibraryUrlRootsDescription(libraryEntity, roots)) + if (!Registry.`is`("use.workspace.file.index.for.partial.scanning")) { + descriptions.add(LibraryUrlRootsDescription(libraryEntity, roots)) + } } for ((entityReference, roots) in rootData.externalRoots.entries) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/roots/ModuleDependencyEntitiesIndexableEntityProvider.java b/platform/lang-impl/src/com/intellij/util/indexing/roots/ModuleDependencyEntitiesIndexableEntityProvider.java index 866f319c718d..463b0e890b32 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/roots/ModuleDependencyEntitiesIndexableEntityProvider.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/roots/ModuleDependencyEntitiesIndexableEntityProvider.java @@ -2,6 +2,7 @@ package com.intellij.util.indexing.roots; import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.registry.Registry; import com.intellij.platform.workspace.jps.entities.*; import com.intellij.util.SmartList; import com.intellij.util.indexing.roots.builders.IndexableIteratorBuilders; @@ -56,6 +57,7 @@ public final class ModuleDependencyEntitiesIndexableEntityProvider implements In ((SdkDependency)dependency).getSdk().getType())); } else if (dependency instanceof LibraryDependency) { + if (Registry.is("use.workspace.file.index.for.partial.scanning")) return Collections.emptyList(); LibraryId libraryId = ((LibraryDependency)dependency).getLibrary(); return IndexableIteratorBuilders.INSTANCE.forLibraryEntity(libraryId, true); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/roots/builders/LibraryIndexableIteratorHandler.kt b/platform/lang-impl/src/com/intellij/util/indexing/roots/builders/LibraryIndexableIteratorHandler.kt index 3c8da3c5b497..9cf2d5db699a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/roots/builders/LibraryIndexableIteratorHandler.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/roots/builders/LibraryIndexableIteratorHandler.kt @@ -3,6 +3,7 @@ package com.intellij.util.indexing.roots.builders import com.intellij.openapi.application.ReadAction import com.intellij.openapi.project.Project +import com.intellij.openapi.util.registry.Registry import com.intellij.openapi.vfs.VirtualFile import com.intellij.platform.workspace.jps.entities.* import com.intellij.platform.workspace.storage.EntityStorage @@ -16,6 +17,7 @@ import com.intellij.workspaceModel.ide.legacyBridge.ModuleDependencyIndex class LibraryIndexableIteratorHandler : IndexableIteratorBuilderHandler { override fun accepts(builder: IndexableEntityProvider.IndexableIteratorBuilder): Boolean = + !Registry.`is`("use.workspace.file.index.for.partial.scanning") && builder is LibraryIdIteratorBuilder override fun instantiate(builders: Collection, diff --git a/platform/lang-impl/src/com/intellij/util/indexing/roots/origin/indexableSetOrigins.kt b/platform/lang-impl/src/com/intellij/util/indexing/roots/origin/indexableSetOrigins.kt index 5a725009cbea..784d7cd1ace2 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/roots/origin/indexableSetOrigins.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/roots/origin/indexableSetOrigins.kt @@ -2,7 +2,6 @@ package com.intellij.util.indexing.roots.origin import com.intellij.openapi.module.Module -import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.roots.SyntheticLibrary import com.intellij.openapi.vfs.VirtualFile import com.intellij.platform.workspace.storage.EntityPointer diff --git a/platform/lang-impl/src/com/intellij/workspaceModel/ide/impl/legacyBridge/project/ProjectRootManagerBridge.kt b/platform/lang-impl/src/com/intellij/workspaceModel/ide/impl/legacyBridge/project/ProjectRootManagerBridge.kt index 6a03c1a62f63..dc8188393f43 100644 --- a/platform/lang-impl/src/com/intellij/workspaceModel/ide/impl/legacyBridge/project/ProjectRootManagerBridge.kt +++ b/platform/lang-impl/src/com/intellij/workspaceModel/ide/impl/legacyBridge/project/ProjectRootManagerBridge.kt @@ -8,6 +8,7 @@ import com.intellij.openapi.roots.impl.OrderRootsCache import com.intellij.openapi.roots.impl.ProjectRootManagerComponent import com.intellij.openapi.roots.libraries.Library import com.intellij.openapi.util.EmptyRunnable +import com.intellij.openapi.util.registry.Registry import com.intellij.platform.workspace.jps.entities.LibraryTableId import com.intellij.util.indexing.BuildableRootsChangeRescanningInfo import com.intellij.workspaceModel.ide.impl.legacyBridge.library.LibraryBridge @@ -55,12 +56,14 @@ class ProjectRootManagerBridge(project: Project, coroutineScope: CoroutineScope) private var insideRootsChange = false override fun referencedLibraryAdded(library: Library) { + if (Registry.`is`("use.workspace.file.index.for.partial.scanning")) return if (shouldListen(library)) { fireRootsChanged(BuildableRootsChangeRescanningInfo.newInstance().addLibrary(library).buildInfo()) } } override fun referencedLibraryChanged(library: Library) { + if (Registry.`is`("use.workspace.file.index.for.partial.scanning")) return if (insideRootsChange || !shouldListen(library)) return insideRootsChange = true try { diff --git a/platform/lang-impl/testSources/com/intellij/util/indexing/EntityIndexingServiceTest.java b/platform/lang-impl/testSources/com/intellij/util/indexing/EntityIndexingServiceTest.java index 752b99c0ea8e..beae72a2a2ed 100644 --- a/platform/lang-impl/testSources/com/intellij/util/indexing/EntityIndexingServiceTest.java +++ b/platform/lang-impl/testSources/com/intellij/util/indexing/EntityIndexingServiceTest.java @@ -12,6 +12,7 @@ import com.intellij.openapi.roots.libraries.Library; import com.intellij.openapi.roots.libraries.LibraryTable; import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar; import com.intellij.openapi.roots.ui.configuration.SdkTestCase; +import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.platform.workspace.jps.entities.LibraryId; import com.intellij.platform.workspace.jps.entities.LibraryTableId; @@ -50,7 +51,9 @@ public class EntityIndexingServiceTest extends EntityIndexingServiceTestBase { } public void testIndexingProjectLibrary() throws Exception { - doTest(this::createProjectLibrary, this::removeProjectLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList); + if (!Registry.is("use.workspace.file.index.for.partial.scanning", false)) { + doTest(this::createProjectLibrary, this::removeProjectLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList); + } } @NotNull @@ -84,8 +87,10 @@ public class EntityIndexingServiceTest extends EntityIndexingServiceTestBase { } public void testIndexingGlobalLibrary() throws Exception { - doTest(this::createGlobalLibrary, this::removeGlobalLibrary, - pair -> LibraryIndexableFilesIteratorImpl.createIteratorList(pair.getFirst())); + if (!Registry.is("use.workspace.file.index.for.partial.scanning", false)) { + doTest(this::createGlobalLibrary, this::removeGlobalLibrary, + pair -> LibraryIndexableFilesIteratorImpl.createIteratorList(pair.getFirst())); + } } @NotNull @@ -100,7 +105,9 @@ public class EntityIndexingServiceTest extends EntityIndexingServiceTestBase { } public void testIndexingModuleLibrary() throws Exception { - doTest(this::createModuleLibrary, this::removeModuleLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList); + if (!Registry.is("use.workspace.file.index.for.partial.scanning", false)) { + doTest(this::createModuleLibrary, this::removeModuleLibrary, LibraryIndexableFilesIteratorImpl::createIteratorList); + } } private void removeModuleLibrary(Library library) { diff --git a/platform/projectModel-impl/resources/META-INF/WorkspaceModelImpl.xml b/platform/projectModel-impl/resources/META-INF/WorkspaceModelImpl.xml index f199dc302f44..991e759e256c 100644 --- a/platform/projectModel-impl/resources/META-INF/WorkspaceModelImpl.xml +++ b/platform/projectModel-impl/resources/META-INF/WorkspaceModelImpl.xml @@ -47,6 +47,8 @@ description="Removes custom processing of Global Libraries and SDKs in the Workspace Model"/> + diff --git a/platform/projectModel-impl/src/com/intellij/workspaceModel/core/fileIndex/impl/libraryRootContributors.kt b/platform/projectModel-impl/src/com/intellij/workspaceModel/core/fileIndex/impl/libraryRootContributors.kt index eb00cd455f37..2cc182cb85de 100644 --- a/platform/projectModel-impl/src/com/intellij/workspaceModel/core/fileIndex/impl/libraryRootContributors.kt +++ b/platform/projectModel-impl/src/com/intellij/workspaceModel/core/fileIndex/impl/libraryRootContributors.kt @@ -10,11 +10,13 @@ import com.intellij.openapi.vfs.VfsUtilCore import com.intellij.openapi.vfs.VirtualFile import com.intellij.openapi.vfs.VirtualFileVisitor import com.intellij.platform.backend.workspace.virtualFile +import com.intellij.platform.workspace.jps.entities.LibraryDependency import com.intellij.platform.workspace.jps.entities.LibraryEntity import com.intellij.platform.workspace.jps.entities.LibraryId import com.intellij.platform.workspace.jps.entities.LibraryRoot.InclusionOptions.* import com.intellij.platform.workspace.jps.entities.LibraryRootTypeId import com.intellij.platform.workspace.jps.entities.LibraryTableId +import com.intellij.platform.workspace.jps.entities.ModuleEntity import com.intellij.platform.workspace.storage.EntityStorage import com.intellij.platform.workspace.storage.url.VirtualFileUrl import com.intellij.util.asSafely @@ -32,6 +34,12 @@ class LibraryRootFileIndexContributor : WorkspaceFileIndexContributor { data = sourceRootFileSetData kind = WorkspaceFileKind.EXTERNAL_SOURCE - } + } else -> continue } when (root.inclusionOptions) { @@ -56,6 +64,13 @@ class LibraryRootFileIndexContributor : WorkspaceFileIndexContributor> + get() = listOf( + DependencyDescription.OnReference(ModuleEntity::class.java) { moduleEntity -> + moduleEntity.dependencies.asSequence().filterIsInstance().map { it.library } + } + ) + private fun registerArchivesUnderRoot(root: VirtualFileUrl, registrar: WorkspaceFileSetRegistrar, data: LibraryRootFileSetData,