IJPL-197024 IJPL-196886 [workspace model] NonIndexableFileNavigationContributor fixes

Cherry-picked from IJ-CR-169167

GitOrigin-RevId: 12639bfd6f247c729fca027cf3d17ec0dc1da5bb
This commit is contained in:
Lev Leontev
2025-07-21 19:53:36 +02:00
committed by intellij-monorepo-bot
parent 0645df19fd
commit f26bf7faaf
8 changed files with 191 additions and 118 deletions

View File

@@ -0,0 +1,120 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:JvmName("NonIndexableFilesUtils")
package com.intellij.util.indexing
import com.intellij.openapi.application.ReadAction
import com.intellij.openapi.application.runReadAction
import com.intellij.openapi.progress.ProgressManager
import com.intellij.openapi.project.Project
import com.intellij.openapi.roots.ContentIterator
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileFilter
import com.intellij.openapi.vfs.VirtualFileVisitor
import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
import com.intellij.util.concurrency.annotations.RequiresReadLock
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileIndex
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileKind
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileSet
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileSetWithCustomData
import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexEx
import org.jetbrains.annotations.ApiStatus
@ApiStatus.Internal
fun iterateNonIndexableFilesImpl(project: Project, inputFilter: VirtualFileFilter?, processor: ContentIterator): Boolean {
val workspaceFileIndex = WorkspaceFileIndexEx.getInstance(project)
val roots: Set<VirtualFile> = ReadAction.nonBlocking<Set<VirtualFile>> { workspaceFileIndex.contentUnindexedRoots() }.executeSynchronously()
return workspaceFileIndex.iterateNonIndexableFilesImpl(roots, inputFilter ?: VirtualFileFilter.ALL, processor)
}
@ApiStatus.Internal
@RequiresBackgroundThread
@RequiresReadLock
private fun WorkspaceFileIndexEx.contentUnindexedRoots(): Set<VirtualFile> {
val roots = mutableSetOf<VirtualFile>()
visitFileSets { fileSet, _ ->
val root = fileSet.root
if (fileSet.kind == WorkspaceFileKind.CONTENT_NON_INDEXABLE) {
roots.add(root)
}
}
return roots
}
private data class AllFileSets(val recursive: List<WorkspaceFileSet>, val nonRecursive: List<WorkspaceFileSet>)
private fun WorkspaceFileIndex.allIndexableFileSets(root: VirtualFile): AllFileSets = runReadAction {
findFileSets(root, true, true, false, true, true, true).partition { fileSet ->
fileSet !is WorkspaceFileSetWithCustomData<*> || fileSet.recursive
}
}.let { (recursive, nonRecursive) -> AllFileSets(recursive, nonRecursive) }
@RequiresBackgroundThread
private fun WorkspaceFileIndex.iterateNonIndexableFilesImpl(roots: Set<VirtualFile>, filter: VirtualFileFilter, processor: ContentIterator): Boolean {
for (root in roots) {
val res = VfsUtilCore.visitChildrenRecursively(root, object : VirtualFileVisitor<Any?>() {
override fun visitFileEx(file: VirtualFile): Result {
ProgressManager.checkCanceled()
val currentIndexableFileSets = allIndexableFileSets(root = file)
return when {
!filter.accept(file) -> SKIP_CHILDREN
currentIndexableFileSets.recursive.isNotEmpty() -> SKIP_CHILDREN
currentIndexableFileSets.nonRecursive.isNotEmpty() -> CONTINUE // skip only the current file, children can be non-indexable
!processor.processFile(file) -> skipTo(root) // terminate processing
else -> CONTINUE
}
}
})
if (res.skipChildren && res.skipToParent == root) return false
}
return true
}
@ApiStatus.Internal
interface FilesDeque {
@RequiresReadLock
fun computeNext(): VirtualFile?
companion object {
/**
* Use [FileBasedIndex.iterateNonIndexableFiles] instead.
*
* This method is only for rare specific use-cases,
* where we need to process non-indexable files in a non-blocking read action, such as find-in-files
*/
@ApiStatus.Internal
@RequiresReadLock
@RequiresBackgroundThread
fun nonIndexableDequeue(project: Project): FilesDeque {
return NonIndexableFilesDequeImpl(project, WorkspaceFileIndexEx.getInstance(project).contentUnindexedRoots())
}
}
}
private class NonIndexableFilesDequeImpl(private val project: Project, private val roots: Set<VirtualFile>) : FilesDeque {
private val bfsQueue: ArrayDeque<VirtualFile> = ArrayDeque(roots)
private val visitedRoots: MutableSet<VirtualFile> = mutableSetOf()
@RequiresReadLock
override fun computeNext(): VirtualFile? {
while (bfsQueue.isNotEmpty()) {
val file = bfsQueue.removeFirst()
if (file in visitedRoots) continue
if (file in roots) visitedRoots.add(file)
val indexableFileSets = WorkspaceFileIndexEx.getInstance(project).allIndexableFileSets(file)
if (indexableFileSets.recursive.isNotEmpty()) continue // skip the current file and their children
if (file.isDirectory) bfsQueue.addAll(file.children)
if (indexableFileSets.nonRecursive.isNotEmpty()) continue // skip only the current file, children can be non-indexable
return file
}
return null
}
}

View File

@@ -55,6 +55,7 @@ import com.intellij.util.containers.ConcurrentBitSet;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.FileBasedIndex;
import com.intellij.util.indexing.FileBasedIndexEx;
import com.intellij.util.indexing.FilesDeque;
import com.intellij.util.indexing.roots.IndexableEntityProviderMethods;
import com.intellij.util.indexing.roots.IndexableFilesIterator;
import com.intellij.util.indexing.roots.kind.ContentOrigin;
@@ -75,7 +76,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Predicate;
import static com.intellij.find.impl.FindInProjectUtil.FIND_IN_FILES_SEARCH_IN_NON_INDEXABLE;
import static com.intellij.find.impl.NonIndexableFilesDequeKt.nonIndexableFiles;
import static com.intellij.openapi.roots.impl.FilesScanExecutor.processOnAllThreadsInReadActionWithRetries;
import static com.intellij.util.containers.ContainerUtil.sorted;
@@ -589,7 +589,7 @@ final class FindInProjectTask {
searchItems.addAll(FindModelExtension.EP_NAME.getExtensionList());
if (Boolean.TRUE.equals(project.getUserData(FIND_IN_FILES_SEARCH_IN_NON_INDEXABLE))) {
searchItems.add(ReadAction.nonBlocking(() -> nonIndexableFiles(project)).executeSynchronously());
searchItems.add(ReadAction.nonBlocking(() -> FilesDeque.Companion.nonIndexableDequeue(project)).executeSynchronously());
}
return searchItems;

View File

@@ -1,53 +0,0 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.find.impl
import com.intellij.ide.util.gotoByName.contentUnindexedRoots
import com.intellij.openapi.project.Project
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.util.concurrency.annotations.RequiresReadLock
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileIndex
import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexEx
import org.jetbrains.annotations.ApiStatus
@ApiStatus.Internal
interface FilesDeque {
@RequiresReadLock
fun computeNext(): VirtualFile?
}
private class NonIndexableFilesDequeImpl(private val roots: Set<VirtualFile>, private val filter: (VirtualFile) -> Boolean) : FilesDeque {
private val bfsQueue: ArrayDeque<VirtualFile> = ArrayDeque(roots)
private val visitedRoots: MutableSet<VirtualFile> = mutableSetOf()
@RequiresReadLock
override fun computeNext(): VirtualFile? {
while (bfsQueue.isNotEmpty()) {
val file = bfsQueue.removeFirst()
if (file in visitedRoots) continue
if (file in roots) visitedRoots.add(file)
if (!filter(file)) continue
if (file.isDirectory) bfsQueue.addAll(file.children)
return file
}
return null
}
}
@ApiStatus.Internal
@RequiresReadLock
fun nonIndexableFiles(project: Project): FilesDeque {
val workspaceFileIndex = WorkspaceFileIndex.getInstance(project) as WorkspaceFileIndexEx
return NonIndexableFilesDequeImpl(workspaceFileIndex.contentUnindexedRoots()) { file ->
!file.isIndexedOrExcluded(workspaceFileIndex)
}
}
private fun VirtualFile.isIndexedOrExcluded(workspaceFileIndex: WorkspaceFileIndexEx): Boolean {
return workspaceFileIndex.isIndexable(this) || !workspaceFileIndex.isInWorkspace(this)
}

View File

@@ -8,16 +8,10 @@ import com.intellij.openapi.project.DumbAware
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.Key
import com.intellij.openapi.util.registry.Registry
import com.intellij.openapi.vfs.VfsUtil
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileVisitor
import com.intellij.psi.search.GlobalSearchScope
import com.intellij.util.Processor
import com.intellij.util.indexing.FileBasedIndex
import com.intellij.util.indexing.FileBasedIndexEx
import com.intellij.util.indexing.FindSymbolParameters
import com.intellij.util.indexing.IdFilter
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileIndex
import com.intellij.util.indexing.*
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileKind
import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexEx
import org.jetbrains.annotations.ApiStatus
@@ -34,37 +28,19 @@ val GOTO_FILE_SEARCH_IN_NON_INDEXABLE: Key<Boolean> = Key.create("search.in.non.
@ApiStatus.Internal
class NonIndexableFileNavigationContributor : ChooseByNameContributorEx, DumbAware {
private inline fun processFileTree(
roots: Set<VirtualFile>,
scope: GlobalSearchScope,
workspaceFileIndex: WorkspaceFileIndexEx,
crossinline processor: (VirtualFile) -> Boolean,
) {
// to avoid processing subtrees multiple times, we process roots first and then do not enter them again
roots.forEach { root -> if (!processor(root)) return }
val rootsChildren = roots.asSequence().flatMap { it.children?.asSequence() ?: emptySequence() }
rootsChildren.forEach { root ->
val res = VfsUtil.visitChildrenRecursively(root, object : VirtualFileVisitor<Any?>() {
override fun visitFileEx(file: VirtualFile): Result = when {
file in roots -> SKIP_CHILDREN
!scope.contains(file) -> SKIP_CHILDREN
file.isIndexedOrExcluded(workspaceFileIndex) -> SKIP_CHILDREN
!processor(file) -> skipTo(root) // terminate processing
else -> CONTINUE
}
})
if (res.skipChildren && res.skipToParent == root) return
}
}
override fun processNames(processor: Processor<in String>, scope: GlobalSearchScope, filter: IdFilter?) {
val project = scope.project ?: return
if (!isGotoFileToNonIndexableEnabled(project)) return
val workspaceFileIndex = WorkspaceFileIndex.getInstance(project) as WorkspaceFileIndexEx
val roots = workspaceFileIndex.contentUnindexedRoots()
processFileTree(roots, scope, workspaceFileIndex) { file -> processor.process(file.name) }
val filenamesProcessed = hashSetOf<String>()
iterateNonIndexableFilesImpl(project, scope, { file ->
val filename = file.name
if (filenamesProcessed.add(filename)) {
processor.process(filename)
}
else {
true
}
})
}

View File

@@ -4,6 +4,7 @@ package com.intellij.util.indexing.testEntities
import com.intellij.platform.workspace.storage.EntityStorage
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileIndexContributor
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileKind
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileSetData
import com.intellij.workspaceModel.core.fileIndex.WorkspaceFileSetRegistrar
class NonIndexableKindFileSetTestContributor : WorkspaceFileIndexContributor<NonIndexableTestEntity> {
@@ -26,3 +27,14 @@ class IndexableKindFileSetTestContributor : WorkspaceFileIndexContributor<Indexi
}
}
}
class NonRecursiveFileSetContributor : WorkspaceFileIndexContributor<NonRecursiveTestEntity> {
override val entityClass: Class<NonRecursiveTestEntity>
get() = NonRecursiveTestEntity::class.java
override fun registerFileSets(entity: NonRecursiveTestEntity, registrar: WorkspaceFileSetRegistrar, storage: EntityStorage) {
registrar.registerNonRecursiveFileSet(entity.root, WorkspaceFileKind.CONTENT, entity, NonRecursiveFileCustomData())
}
}
class NonRecursiveFileCustomData : WorkspaceFileSetData

View File

@@ -3,7 +3,6 @@ package com.intellij.find
import com.intellij.find.impl.FindInProjectUtil
import com.intellij.find.impl.FindInProjectUtil.FIND_IN_FILES_SEARCH_IN_NON_INDEXABLE
import com.intellij.find.impl.nonIndexableFiles
import com.intellij.openapi.Disposable
import com.intellij.openapi.application.readAction
import com.intellij.openapi.vfs.VirtualFile
@@ -18,10 +17,8 @@ import com.intellij.testFramework.rules.ProjectModelExtension
import com.intellij.usageView.UsageInfo
import com.intellij.util.CommonProcessors
import com.intellij.util.Processor
import com.intellij.util.indexing.testEntities.IndexableKindFileSetTestContributor
import com.intellij.util.indexing.testEntities.IndexingTestEntity
import com.intellij.util.indexing.testEntities.NonIndexableKindFileSetTestContributor
import com.intellij.util.indexing.testEntities.NonIndexableTestEntity
import com.intellij.util.indexing.FilesDeque
import com.intellij.util.indexing.testEntities.*
import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexImpl
import com.intellij.workspaceModel.ide.NonPersistentEntitySource
import kotlinx.coroutines.runBlocking
@@ -30,6 +27,9 @@ import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.RegisterExtension
import java.util.*
/**
* Run with `-cp intellij.idea.ultimate.test.main`
*/
@TestApplication
class SearchInNonIndexableTest() {
@RegisterExtension
@@ -47,6 +47,7 @@ class SearchInNonIndexableTest() {
fun setup(): Unit = runBlocking {
WorkspaceFileIndexImpl.EP_NAME.point.registerExtension(NonIndexableKindFileSetTestContributor(), disposable)
WorkspaceFileIndexImpl.EP_NAME.point.registerExtension(IndexableKindFileSetTestContributor(), disposable)
WorkspaceFileIndexImpl.EP_NAME.point.registerExtension(NonRecursiveFileSetContributor(), disposable)
project.putUserData(FIND_IN_FILES_SEARCH_IN_NON_INDEXABLE, true)
@@ -58,9 +59,13 @@ class SearchInNonIndexableTest() {
val indexable = baseDir.newVirtualDirectory("indexable").toVirtualFileUrl(urlManager)
baseDir.newVirtualFile("indexable/infile1", "this is a file with some data and indexes".toByteArray())
val indexableNonRecursive = baseDir.newVirtualDirectory("non-indexable/indexable-non-recursive").toVirtualFileUrl(urlManager)
baseDir.newVirtualFile("non-indexable/indexable-non-recursive/non-indexable-beats-non-recursive-content", "this is a file with some data".toByteArray())
project.workspaceModel.update("add non-indexable root") { storage ->
storage.addEntity(NonIndexableTestEntity(nonIndexable, NonPersistentEntitySource))
storage.addEntity(IndexingTestEntity(listOf(indexable), emptyList(), NonPersistentEntitySource))
storage.addEntity(NonRecursiveTestEntity(indexableNonRecursive, NonPersistentEntitySource))
}
VfsTestUtil.syncRefresh()
waitUntilIndexesAreReady(project)
@@ -68,14 +73,14 @@ class SearchInNonIndexableTest() {
@Test
fun `non-indexable files deque`(): Unit = runBlocking {
val deque = readAction {nonIndexableFiles(project)}
val deque = readAction { FilesDeque.nonIndexableDequeue(project)}
val files = mutableListOf<VirtualFile>()
while (true) {
val file = readAction { deque.computeNext() } ?: break
files.add(file)
}
val names = files.map { it.name }
assertThat(names).containsExactlyInAnyOrder("non-indexable", "file1", "file2")
assertThat(names).containsExactlyInAnyOrder("non-indexable", "file1", "file2", "non-indexable-beats-non-recursive-content")
}
@Test
@@ -88,7 +93,7 @@ class SearchInNonIndexableTest() {
FindInProjectUtil.findUsages(model, project, consumer, presentation)
val fileNames = usages.map { it!!.virtualFile!!.name }
assertThat(fileNames).containsExactlyInAnyOrder("file1", "infile1")
assertThat(fileNames).containsExactlyInAnyOrder("file1", "infile1", "non-indexable-beats-non-recursive-content")
}

View File

@@ -21,10 +21,7 @@ import com.intellij.testFramework.junit5.TestDisposable
import com.intellij.testFramework.rules.ProjectModelExtension
import com.intellij.testFramework.workspaceModel.update
import com.intellij.util.CommonProcessors
import com.intellij.util.indexing.testEntities.IndexableKindFileSetTestContributor
import com.intellij.util.indexing.testEntities.IndexingTestEntity
import com.intellij.util.indexing.testEntities.NonIndexableKindFileSetTestContributor
import com.intellij.util.indexing.testEntities.NonIndexableTestEntity
import com.intellij.util.indexing.testEntities.*
import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexImpl
import com.intellij.workspaceModel.ide.NonPersistentEntitySource
import com.intellij.workspaceModel.ide.toPath
@@ -57,6 +54,7 @@ class NonIndexableFileNavigationContributorTest {
fun setUp() {
WorkspaceFileIndexImpl.EP_NAME.point.registerExtension(NonIndexableKindFileSetTestContributor(), disposable)
WorkspaceFileIndexImpl.EP_NAME.point.registerExtension(IndexableKindFileSetTestContributor(), disposable)
WorkspaceFileIndexImpl.EP_NAME.point.registerExtension(NonRecursiveFileSetContributor(), disposable)
}
@@ -141,6 +139,35 @@ class NonIndexableFileNavigationContributorTest {
assertThat(names).containsExactlyInAnyOrder("u1", "u2", "justDir", "f")
}
@Test
fun `indexable non-recursive file set inside non-indexable`(): Unit = runBlocking {
val nonIndexable = baseDir.newVirtualDirectory("non-indexable").toVirtualFileUrl(urlManager)
val indexableNonRecursive = baseDir.newVirtualDirectory("non-indexable/indexable-non-recursive").toVirtualFileUrl(urlManager)
baseDir.newVirtualFile("non-indexable/indexable-non-recursive/file.txt")
workspaceModel.update { storage ->
storage.addEntity(NonIndexableTestEntity(nonIndexable, NonPersistentEntitySource))
storage.addEntity(NonRecursiveTestEntity(indexableNonRecursive, NonPersistentEntitySource))
}
val names = processNames()
assertThat(names).containsExactlyInAnyOrder("file.txt", "non-indexable")
}
@Test
fun `unindexed and non-recursive file set at the same level`(): Unit = runBlocking {
val root = baseDir.newVirtualDirectory("root").toVirtualFileUrl(urlManager)
baseDir.newVirtualFile("root/file.txt")
workspaceModel.update { storage ->
storage.addEntity(NonIndexableTestEntity(root, NonPersistentEntitySource))
storage.addEntity(NonRecursiveTestEntity(root, NonPersistentEntitySource))
}
val names = processNames()
assertThat(names).containsExactlyInAnyOrder("file.txt") // `root` is excluded because it's under non-recursive content root
}
@Test
@DisabledOnOs(OS.WINDOWS)
fun `symlink to file`(): Unit = runBlocking {
@@ -171,7 +198,7 @@ class NonIndexableFileNavigationContributorTest {
VfsTestUtil.syncRefresh()
val names = processNames()
assertThat(names).containsExactlyInAnyOrder("u1", "u2", "link-1", "link-1", "link-2", "link-2")
assertThat(names).containsExactlyInAnyOrder("u1", "u2", "link-1", "link-2")
}
@Test

View File

@@ -10,15 +10,12 @@ import com.intellij.openapi.roots.ModuleRootModificationUtil
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.platform.backend.workspace.WorkspaceModel
import com.intellij.platform.backend.workspace.toVirtualFileUrl
import com.intellij.platform.workspace.storage.EntityStorage
import com.intellij.testFramework.PsiTestUtil
import com.intellij.testFramework.junit5.TestApplication
import com.intellij.testFramework.junit5.TestDisposable
import com.intellij.testFramework.rules.ProjectModelExtension
import com.intellij.testFramework.workspaceModel.update
import com.intellij.util.indexing.testEntities.IndexableKindFileSetTestContributor
import com.intellij.util.indexing.testEntities.IndexingTestEntity
import com.intellij.util.indexing.testEntities.NonRecursiveTestEntity
import com.intellij.util.indexing.testEntities.*
import com.intellij.workspaceModel.core.fileIndex.impl.WorkspaceFileIndexImpl
import com.intellij.workspaceModel.ide.NonPersistentEntitySource
import io.kotest.common.runBlocking
@@ -125,15 +122,4 @@ class NonRecursiveWorkspaceFileSetTest {
}
}
private class NonRecursiveFileSetContributor : WorkspaceFileIndexContributor<NonRecursiveTestEntity> {
override val entityClass: Class<NonRecursiveTestEntity>
get() = NonRecursiveTestEntity::class.java
override fun registerFileSets(entity: NonRecursiveTestEntity, registrar: WorkspaceFileSetRegistrar, storage: EntityStorage) {
registrar.registerNonRecursiveFileSet(entity.root, WorkspaceFileKind.CONTENT, entity, NonRecursiveFileCustomData())
}
}
private class NonRecursiveFileCustomData : WorkspaceFileSetData
}