diff --git a/platform/indexing-api/src/com/intellij/util/indexing/hints/FileTypeIndexingHint.kt b/platform/indexing-api/src/com/intellij/util/indexing/hints/FileTypeIndexingHint.kt new file mode 100644 index 000000000000..419064ca6f20 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/util/indexing/hints/FileTypeIndexingHint.kt @@ -0,0 +1,85 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.util.indexing.hints + +import com.intellij.openapi.fileTypes.FileType +import com.intellij.util.ThreeState +import com.intellij.util.indexing.FileBasedIndex +import com.intellij.util.indexing.FileBasedIndex.InputFilter +import com.intellij.util.indexing.IndexedFile +import org.jetbrains.annotations.ApiStatus.Experimental + +/** + * Provides a hint to indexing framework if this index accepts some specific filetype. + * + * Examples: + * * JsonSchemaFileValuesIndex filetype has a number of subtypes. This hint is as efficient as [FileBasedIndex.FileTypeSpecificInputFilter], + * but more convenient, because implementation can naturally use `instanceOf`, instead of explicitly listing all the known inheritors + * (see [FileTypeInputFilterPredicate]). + * + * * HtmlScriptSrcIndex does not know all the applicable filetypes in advance, but wants to accept any filetype associated with TemplateLanguage. + * + * * text-based indexers with no specific filetypes can quickly reject binary files (see [NonBinaryFileTypeInputFilter]) + * + * See general rules in javadoc for [IndexingHint] + */ +@Experimental +interface FileTypeIndexingHint : IndexingHint { + fun hintAcceptFileType(fileType: FileType): ThreeState +} + +@Experimental +interface FileTypeProjectSpecificInputFilter : FileBasedIndex.ProjectSpecificInputFilter, FileTypeIndexingHint + +/** + * Returns `YES` or `NO` for given filetype predicate. Never returns `UNSURE`, therefore `acceptInput` is never invoked. + */ +@Experimental +class FileTypeInputFilterPredicate(private val predicate: (filetype: FileType) -> Boolean) : BaseFileTypeInputFilter() { + override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean { + throw AssertionError("Should not be invoked, because hintAcceptFileType for filetype never returns UNSURE"); + } + + override fun hintAcceptFileType(fileType: FileType): ThreeState = ThreeState.fromBoolean(predicate(fileType)) +} + +/** + * Returns `NO` for binary file types, and `UNSURE` for others (i.e. delegates to [whenAllOtherHintsUnsure]). + */ +@Experimental +class NonBinaryFileTypeInputFilter(private val acceptInput: InputFilter) : BaseFileTypeInputFilter() { + override fun hintAcceptFileType(fileType: FileType): ThreeState { + return if (fileType.isBinary) ThreeState.NO else ThreeState.UNSURE; + } + + override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean { + return acceptInput.acceptInput(file.file) + } +} + +/** + * Base class for FileTypeInputFilter. Contains default implementation of `acceptInput(file: VirtualFile)` which delegate to hints. + */ +@Experimental +abstract class BaseFileTypeInputFilter : FileTypeProjectSpecificInputFilter { + final override fun acceptInput(file: IndexedFile): Boolean { + return when (hintAcceptFileType(file.fileType)) { + ThreeState.YES -> true + ThreeState.NO -> false + ThreeState.UNSURE -> whenAllOtherHintsUnsure(file) + } + } +} + +/** + * Base class for FileTypeInputFilter. Contains default implementation of `acceptInput(file: VirtualFile)` which delegate to hints. + */ +@Experimental +abstract class BaseFileTypeProjectSpecificInputFilter : FileTypeProjectSpecificInputFilter { + final override fun acceptInput(file: IndexedFile): Boolean { + return when (hintAcceptFileType(file.fileType)) { + ThreeState.YES -> true + ThreeState.NO -> false + ThreeState.UNSURE -> whenAllOtherHintsUnsure(file) + } + } +} \ No newline at end of file diff --git a/platform/indexing-api/src/com/intellij/util/indexing/hints/IndexingHint.kt b/platform/indexing-api/src/com/intellij/util/indexing/hints/IndexingHint.kt new file mode 100644 index 000000000000..7a525d58e798 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/util/indexing/hints/IndexingHint.kt @@ -0,0 +1,55 @@ +// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.util.indexing.hints + +import com.intellij.util.ThreeState +import com.intellij.util.indexing.FileBasedIndex.InputFilter +import com.intellij.util.indexing.FileBasedIndex.ProjectSpecificInputFilter +import com.intellij.util.indexing.IndexedFile + +/** + * **General rules applicable to any hint:** + * + * All the hints are evaluated before [IndexingHint.whenAllOtherHintsUnsure]. It is guaranteed that [IndexingHint.whenAllOtherHintsUnsure] + * is only evaluated for files for which all hints returned [ThreeState.UNSURE]. I.e. there is no need to add the logic from hints to + * [IndexingHint.whenAllOtherHintsUnsure]. But this logic still should be added to [InputFilter.acceptInput] as explained below. + * + * If a hint returns either [ThreeState.YES] or [ThreeState.NO], neither [IndexingHint.whenAllOtherHintsUnsure] nor other hints are evaluated. + * + * Hint results are cached (at least until IDE restart). In particular, if hint uses ExtensionPoints to evaluate result, changes + * in relevant extension points (e.g. loading/unloading plugins) should reset caches. + * Use [com.intellij.util.indexing.FileBasedIndexEx.resetHints] to reset cached indexing hints. + * + * Hints should not assume that they are evaluated in any specific order. + * + * If a hint returns [ThreeState.UNSURE], the indexing framework will first evaluate remaining hints (if any), if all of them return + * [ThreeState.UNSURE], then [IndexingHint.whenAllOtherHintsUnsure] will be invoked for individual files. + * + * The main goal of the hints framework is to avoid iterating the whole VFS on startup just to evaluate if particular + * file should be indexed by a particular indexer. That's why we try to shift the focus from individual files to large group of files. + * + * **When used with [InputFilter] or [ProjectSpecificInputFilter]:** + * + * Indexing framework evaluates all the hints (if available), and falls back to [IndexingHint.whenAllOtherHintsUnsure] if no hint + * returned `YES` or `NO`. [IndexingHint.whenAllOtherHintsUnsure] must answer either `true` or `false`. This means that indexing framework + * will not invoke [InputFilter.acceptInput] or [ProjectSpecificInputFilter.acceptInput]. However, there may be other clients which may + * invoke `acceptInput` without analyzing any hints. Therefore, `acceptInput` should provide answer just like if the filter didn't have any + * hints in the first place. + * + * When using hint-aware base classes (like [FileTypeInputFilterPredicate]) these classes implement `acceptInput` methods for you. + * Usually you don't need to override this method, default implementation boils down to a code like: + * + * ``` + * fun acceptInput(file: IndexedFile): Boolean { + * val res = hintSomethingMethod(...) + * return if (res == ThreeState.UNSURE) whenAllHintsUnsure(file) else fromBoolean(res) + * } + * ``` + * + * **When used with [com.intellij.util.indexing.GlobalIndexFilter]:** + * + * Same as when used with [InputFilter] or [ProjectSpecificInputFilter]. This section is mostly to draw your attention that hints can be + * used with [com.intellij.util.indexing.GlobalIndexFilter] + */ +interface IndexingHint { + fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean; +} \ No newline at end of file diff --git a/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java b/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java index 26a6616f4785..5bd7dbff33f6 100644 --- a/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java +++ b/platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java @@ -105,6 +105,8 @@ public abstract class FileBasedIndexEx extends FileBasedIndex { @ApiStatus.Internal public abstract UpdatableIndex getIndex(ID indexId); + public void resetHints() { } + @ApiStatus.Internal public abstract void waitUntilIndicesAreInitialized(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index ce834b263df7..9ede7b7b9d02 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -436,6 +436,11 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx { } } + @Override + public void resetHints() { + myRegisteredIndexes.resetHints(); + } + @Override public void waitUntilIndicesAreInitialized() { if (myRegisteredIndexes == null) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/RegisteredIndexes.java b/platform/lang-impl/src/com/intellij/util/indexing/RegisteredIndexes.java index 73b2b076b163..bbc7bb8a83d4 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/RegisteredIndexes.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/RegisteredIndexes.java @@ -100,6 +100,10 @@ public final class RegisteredIndexes { myRequiredIndexesEvaluator = new RequiredIndexesEvaluator(this); } + void resetHints() { + myRequiredIndexesEvaluator = new RequiredIndexesEvaluator(this); + } + void ensureLoadedIndexesUpToDate() { myAllIndicesInitializedFuture = IndexDataInitializer.submitGenesisTask(() -> { if (!myShutdownPerformed.get()) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/RequiredIndexesEvaluator.kt b/platform/lang-impl/src/com/intellij/util/indexing/RequiredIndexesEvaluator.kt index ef0ac90b8376..5cb113deeb90 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/RequiredIndexesEvaluator.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/RequiredIndexesEvaluator.kt @@ -1,7 +1,11 @@ // Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.util.indexing +import com.intellij.openapi.diagnostic.logger import com.intellij.openapi.fileTypes.FileType +import com.intellij.util.ThreeState +import com.intellij.util.indexing.hints.FileTypeIndexingHint +import com.intellij.util.indexing.hints.IndexingHint import com.jetbrains.rd.util.concurrentMapOf import java.util.function.Predicate @@ -12,8 +16,14 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere override fun test(t: IndexedFile): Boolean = true } - private fun combinePredicates(p1: IndexedFilePredicate, p2: IndexedFilePredicate): IndexedFilePredicate { - if (p1 == truePredicate) return p2 + private val falsePredicate = object : IndexedFilePredicate { + override fun test(t: IndexedFile): Boolean = false + } + + private fun andPredicates(p1: IndexedFilePredicate, p2: IndexedFilePredicate): IndexedFilePredicate { + if (p1 == falsePredicate) return p1 + else if (p2 == falsePredicate) return p2 + else if (p1 == truePredicate) return p2 else if (p2 == truePredicate) return p1 else return object : IndexedFilePredicate { override fun test(t: IndexedFile): Boolean = p1.test(t) && p2.test(t) @@ -28,21 +38,21 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere FileBasedIndexImpl.LOG.assertTrue(registeredIndexes.isInitialized, "RegisteredIndexes are not initialized") } - constructor(unsureIndexIds: Collection>) { - val sure: MutableList> = mutableListOf>() + constructor(indexIds: Collection>, fileType: FileType? = null) { + val sure: MutableList> = mutableListOf() val unsure: MutableList, IndexedFilePredicate>> = mutableListOf() this.sureIndexIds = sure this.unsureIndexIds = unsure - - for (unsureIndexId in unsureIndexIds) { - val predicate = acceptsInput(unsureIndexId) - if (predicate == truePredicate) sure.add(unsureIndexId) - else unsure.add(Pair(unsureIndexId, predicate)) + for (indexId in indexIds) { + val predicate = acceptsInput(indexId, fileType) + if (predicate == truePredicate) sure.add(indexId) + else if (predicate != falsePredicate) unsure.add(Pair(indexId, predicate)) } + LOG.debug("fileType: $fileType, slow scanning path via indexes: ${unsure.map { it.first.name }.toList()}") } fun getRequiredIndexes(indexedFile: IndexedFile): List> { - if (unsureIndexIds.isEmpty()) return sureIndexIds; + if (unsureIndexIds.isEmpty()) return sureIndexIds FileBasedIndexImpl.LOG.assertTrue(indexedFile.project != null, "Should not index files from unknown project") val acceptedCandidates: MutableList> = ArrayList(sureIndexIds) @@ -61,22 +71,47 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere private fun getState(): IndexConfiguration = registeredIndexes.configurationState private fun getInputFilter(indexId: ID<*, *>): FileBasedIndex.InputFilter = getState().getInputFilter(indexId) - private fun acceptsInput(indexId: ID<*, *>): IndexedFilePredicate { - val indexAccepts = getIndexerFilter(indexId) - val globalAccepts = getGlobalFilter(indexId) - return combinePredicates(indexAccepts, globalAccepts) + private fun acceptsInput(indexId: ID<*, *>, fileType: FileType?): IndexedFilePredicate { + val hint = toHint(getInputFilter(indexId)) + val globalHint = getGlobalHint(indexId) + + val indexerHintPredicate = applyHints(hint, fileType) + val globalHintPredicate = applyHints(globalHint, fileType) + + return andPredicates(indexerHintPredicate, globalHintPredicate) } - private fun getIndexerFilter(indexId: ID<*, *>) = object : IndexedFilePredicate { - override fun test(indexedFile: IndexedFile): Boolean { - val filter: FileBasedIndex.InputFilter = getInputFilter(indexId) - return FileBasedIndexEx.acceptsInput(filter, indexedFile) + private fun toHint(filter: FileBasedIndex.InputFilter): IndexingHint { + return (filter as? IndexingHint) ?: object : IndexingHint { + override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean { + return FileBasedIndexEx.acceptsInput(filter, file) + } } } - private fun getGlobalFilter(indexId: ID<*, *>) = object : IndexedFilePredicate { - override fun test(indexedFile: IndexedFile): Boolean { - return !GlobalIndexFilter.isExcludedFromIndexViaFilters(indexedFile.file, indexId, indexedFile.project) + private fun getGlobalHint(indexId: ID<*, *>): IndexingHint = object : IndexingHint { + override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean { + return !GlobalIndexFilter.isExcludedFromIndexViaFilters(file.file, indexId, file.project) + } + } + + private fun applyHints(indexingHint: IndexingHint, fileType: FileType?): IndexedFilePredicate { + val hint = applyFileTypeHint(indexingHint, fileType) + return when (hint) { + ThreeState.YES -> truePredicate + ThreeState.NO -> falsePredicate + ThreeState.UNSURE -> object : IndexedFilePredicate { + override fun test(indexedFile: IndexedFile): Boolean = indexingHint.whenAllOtherHintsUnsure(indexedFile) + } + } + } + + private fun applyFileTypeHint(indexingHint: IndexingHint, fileType: FileType?): ThreeState { + if (fileType != null && indexingHint is FileTypeIndexingHint) { + return indexingHint.hintAcceptFileType(fileType) + } + else { + return ThreeState.UNSURE } } @@ -102,7 +137,7 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere var filteredResults = indexesForFileType[fileType] if (filteredResults != null) return filteredResults.getRequiredIndexes(indexedFile) - filteredResults = HintAwareIndexList(getState().getFileTypesForIndex(fileType)) + filteredResults = HintAwareIndexList(getState().getFileTypesForIndex(fileType), fileType) indexesForFileType[fileType] = filteredResults return filteredResults.getRequiredIndexes(indexedFile) } @@ -116,4 +151,8 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere return indexesForDirectories.getRequiredIndexes(indexedFile) } } + + companion object { + private val LOG = logger() + } } \ No newline at end of file