IJPL-27: introduce IndexingHint framework in general and FileTypeIndexingHint in particular

GitOrigin-RevId: b79ff5ae946f01f619a4969fa0b4748aa7584c6d

platform/indexing-api/src/com/intellij/util/indexing/hints/FileTypeIndexingHint.kt

@@ -0,0 +1,85 @@
+// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.util.indexing.hints
+
+import com.intellij.openapi.fileTypes.FileType
+import com.intellij.util.ThreeState
+import com.intellij.util.indexing.FileBasedIndex
+import com.intellij.util.indexing.FileBasedIndex.InputFilter
+import com.intellij.util.indexing.IndexedFile
+import org.jetbrains.annotations.ApiStatus.Experimental
+
+/**
+ * Provides a hint to indexing framework if this index accepts some specific filetype.
+ *
+ * Examples:
+ *   * JsonSchemaFileValuesIndex filetype has a number of subtypes. This hint is as efficient as [FileBasedIndex.FileTypeSpecificInputFilter],
+ *   but more convenient, because implementation can naturally use `instanceOf`, instead of explicitly listing all the known inheritors
+ *   (see [FileTypeInputFilterPredicate]).
+ *
+ *   * HtmlScriptSrcIndex does not know all the applicable filetypes in advance, but wants to accept any filetype associated with TemplateLanguage.
+ *
+ *   * text-based indexers with no specific filetypes can quickly reject binary files (see [NonBinaryFileTypeInputFilter])
+ *
+ * See general rules in javadoc for [IndexingHint]
+ */
+@Experimental
+interface FileTypeIndexingHint : IndexingHint {
+  fun hintAcceptFileType(fileType: FileType): ThreeState
+}
+
+@Experimental
+interface FileTypeProjectSpecificInputFilter : FileBasedIndex.ProjectSpecificInputFilter, FileTypeIndexingHint
+
+/**
+ * Returns `YES` or `NO` for given filetype predicate. Never returns `UNSURE`, therefore `acceptInput` is never invoked.
+ */
+@Experimental
+class FileTypeInputFilterPredicate(private val predicate: (filetype: FileType) -> Boolean) : BaseFileTypeInputFilter() {
+  override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean {
+    throw AssertionError("Should not be invoked, because hintAcceptFileType for filetype never returns UNSURE");
+  }
+
+  override fun hintAcceptFileType(fileType: FileType): ThreeState = ThreeState.fromBoolean(predicate(fileType))
+}
+
+/**
+ * Returns `NO` for binary file types, and `UNSURE` for others (i.e. delegates to [whenAllOtherHintsUnsure]).
+ */
+@Experimental
+class NonBinaryFileTypeInputFilter(private val acceptInput: InputFilter) : BaseFileTypeInputFilter() {
+  override fun hintAcceptFileType(fileType: FileType): ThreeState {
+    return if (fileType.isBinary) ThreeState.NO else ThreeState.UNSURE;
+  }
+
+  override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean {
+    return acceptInput.acceptInput(file.file)
+  }
+}
+
+/**
+ * Base class for FileTypeInputFilter. Contains default implementation of `acceptInput(file: VirtualFile)` which delegate to hints.
+ */
+@Experimental
+abstract class BaseFileTypeInputFilter : FileTypeProjectSpecificInputFilter {
+  final override fun acceptInput(file: IndexedFile): Boolean {
+    return when (hintAcceptFileType(file.fileType)) {
+      ThreeState.YES -> true
+      ThreeState.NO -> false
+      ThreeState.UNSURE -> whenAllOtherHintsUnsure(file)
+    }
+  }
+}
+
+/**
+ * Base class for FileTypeInputFilter. Contains default implementation of `acceptInput(file: VirtualFile)` which delegate to hints.
+ */
+@Experimental
+abstract class BaseFileTypeProjectSpecificInputFilter : FileTypeProjectSpecificInputFilter {
+  final override fun acceptInput(file: IndexedFile): Boolean {
+    return when (hintAcceptFileType(file.fileType)) {
+      ThreeState.YES -> true
+      ThreeState.NO -> false
+      ThreeState.UNSURE -> whenAllOtherHintsUnsure(file)
+    }
+  }
+}

platform/indexing-api/src/com/intellij/util/indexing/hints/IndexingHint.kt

@@ -0,0 +1,55 @@
+// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.util.indexing.hints
+
+import com.intellij.util.ThreeState
+import com.intellij.util.indexing.FileBasedIndex.InputFilter
+import com.intellij.util.indexing.FileBasedIndex.ProjectSpecificInputFilter
+import com.intellij.util.indexing.IndexedFile
+
+/**
+ *  **General rules applicable to any hint:**
+ *
+ * All the hints are evaluated before [IndexingHint.whenAllOtherHintsUnsure]. It is guaranteed that [IndexingHint.whenAllOtherHintsUnsure]
+ * is only evaluated for files for which all hints returned [ThreeState.UNSURE]. I.e. there is no need to add the logic from hints to
+ * [IndexingHint.whenAllOtherHintsUnsure]. But this logic still should be added to [InputFilter.acceptInput] as explained below.
+ *
+ * If a hint returns either [ThreeState.YES] or [ThreeState.NO], neither [IndexingHint.whenAllOtherHintsUnsure] nor other hints are evaluated.
+ *
+ * Hint results are cached (at least until IDE restart). In particular, if hint uses ExtensionPoints to evaluate result, changes
+ * in relevant extension points (e.g. loading/unloading plugins) should reset caches.
+ * Use [com.intellij.util.indexing.FileBasedIndexEx.resetHints] to reset cached indexing hints.
+ *
+ * Hints should not assume that they are evaluated in any specific order.
+ *
+ * If a hint returns [ThreeState.UNSURE], the indexing framework will first evaluate remaining hints (if any), if all of them return
+ * [ThreeState.UNSURE], then [IndexingHint.whenAllOtherHintsUnsure] will be invoked for individual files.
+ *
+ * The main goal of the hints framework is to avoid iterating the whole VFS on startup just to evaluate if particular
+ * file should be indexed by a particular indexer. That's why we try to shift the focus from individual files to large group of files.
+ *
+ * **When used with [InputFilter] or [ProjectSpecificInputFilter]:**
+ *
+ * Indexing framework evaluates all the hints (if available), and falls back to [IndexingHint.whenAllOtherHintsUnsure] if no hint
+ * returned `YES` or `NO`. [IndexingHint.whenAllOtherHintsUnsure] must answer either `true` or `false`. This means that indexing framework
+ * will not invoke [InputFilter.acceptInput] or [ProjectSpecificInputFilter.acceptInput]. However, there may be other clients which may
+ * invoke `acceptInput` without analyzing any hints. Therefore, `acceptInput` should provide answer just like if the filter didn't have any
+ * hints in the first place.
+ *
+ * When using hint-aware base classes (like [FileTypeInputFilterPredicate]) these classes implement `acceptInput` methods for you.
+ * Usually you don't need to override this method, default implementation boils down to a code like:
+ *
+ * ```
+ * fun acceptInput(file: IndexedFile): Boolean {
+ *   val res = hintSomethingMethod(...)
+ *   return if (res == ThreeState.UNSURE) whenAllHintsUnsure(file) else fromBoolean(res)
+ * }
+ * ```
+ *
+ * **When used with [com.intellij.util.indexing.GlobalIndexFilter]:**
+ *
+ * Same as when used with [InputFilter] or [ProjectSpecificInputFilter]. This section is mostly to draw your attention that hints can be
+ * used with [com.intellij.util.indexing.GlobalIndexFilter]
+ */
+interface IndexingHint {
+  fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean;
+}

platform/indexing-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java

@@ -105,6 +105,8 @@ public abstract class FileBasedIndexEx extends FileBasedIndex {
   @ApiStatus.Internal
   public abstract <K, V> UpdatableIndex<K, V, FileContent, ?> getIndex(ID<K, V> indexId);
 
+  public void resetHints() { }
+
   @ApiStatus.Internal
   public abstract void waitUntilIndicesAreInitialized();
 

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java

@@ -436,6 +436,11 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx {
     }
   }
 
+  @Override
+  public void resetHints() {
+    myRegisteredIndexes.resetHints();
+  }
+
   @Override
   public void waitUntilIndicesAreInitialized() {
     if (myRegisteredIndexes == null) {

platform/lang-impl/src/com/intellij/util/indexing/RegisteredIndexes.java

@@ -100,6 +100,10 @@ public final class RegisteredIndexes {
     myRequiredIndexesEvaluator = new RequiredIndexesEvaluator(this);
   }
 
+  void resetHints() {
+    myRequiredIndexesEvaluator = new RequiredIndexesEvaluator(this);
+  }
+
   void ensureLoadedIndexesUpToDate() {
     myAllIndicesInitializedFuture = IndexDataInitializer.submitGenesisTask(() -> {
       if (!myShutdownPerformed.get()) {

platform/lang-impl/src/com/intellij/util/indexing/RequiredIndexesEvaluator.kt

@@ -1,7 +1,11 @@
 // Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.util.indexing
 
+import com.intellij.openapi.diagnostic.logger
 import com.intellij.openapi.fileTypes.FileType
+import com.intellij.util.ThreeState
+import com.intellij.util.indexing.hints.FileTypeIndexingHint
+import com.intellij.util.indexing.hints.IndexingHint
 import com.jetbrains.rd.util.concurrentMapOf
 import java.util.function.Predicate
 
@@ -12,8 +16,14 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere
     override fun test(t: IndexedFile): Boolean = true
   }
 
-  private fun combinePredicates(p1: IndexedFilePredicate, p2: IndexedFilePredicate): IndexedFilePredicate {
-    if (p1 == truePredicate) return p2
+  private val falsePredicate = object : IndexedFilePredicate {
+    override fun test(t: IndexedFile): Boolean = false
+  }
+
+  private fun andPredicates(p1: IndexedFilePredicate, p2: IndexedFilePredicate): IndexedFilePredicate {
+    if (p1 == falsePredicate) return p1
+    else if (p2 == falsePredicate) return p2
+    else if (p1 == truePredicate) return p2
     else if (p2 == truePredicate) return p1
     else return object : IndexedFilePredicate {
       override fun test(t: IndexedFile): Boolean = p1.test(t) && p2.test(t)
@@ -28,21 +38,21 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere
       FileBasedIndexImpl.LOG.assertTrue(registeredIndexes.isInitialized, "RegisteredIndexes are not initialized")
     }
 
-    constructor(unsureIndexIds: Collection<ID<*, *>>) {
-      val sure: MutableList<ID<*, *>> = mutableListOf<ID<*, *>>()
+    constructor(indexIds: Collection<ID<*, *>>, fileType: FileType? = null) {
+      val sure: MutableList<ID<*, *>> = mutableListOf()
       val unsure: MutableList<Pair<ID<*, *>, IndexedFilePredicate>> = mutableListOf()
       this.sureIndexIds = sure
       this.unsureIndexIds = unsure
-
-      for (unsureIndexId in unsureIndexIds) {
-        val predicate = acceptsInput(unsureIndexId)
-        if (predicate == truePredicate) sure.add(unsureIndexId)
-        else unsure.add(Pair(unsureIndexId, predicate))
+      for (indexId in indexIds) {
+        val predicate = acceptsInput(indexId, fileType)
+        if (predicate == truePredicate) sure.add(indexId)
+        else if (predicate != falsePredicate) unsure.add(Pair(indexId, predicate))
       }
+      LOG.debug("fileType: $fileType, slow scanning path via indexes: ${unsure.map { it.first.name }.toList()}")
     }
 
     fun getRequiredIndexes(indexedFile: IndexedFile): List<ID<*, *>> {
-      if (unsureIndexIds.isEmpty()) return sureIndexIds;
+      if (unsureIndexIds.isEmpty()) return sureIndexIds
 
       FileBasedIndexImpl.LOG.assertTrue(indexedFile.project != null, "Should not index files from unknown project")
       val acceptedCandidates: MutableList<ID<*, *>> = ArrayList(sureIndexIds)
@@ -61,22 +71,47 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere
   private fun getState(): IndexConfiguration = registeredIndexes.configurationState
   private fun getInputFilter(indexId: ID<*, *>): FileBasedIndex.InputFilter = getState().getInputFilter(indexId)
 
-  private fun acceptsInput(indexId: ID<*, *>): IndexedFilePredicate {
-    val indexAccepts = getIndexerFilter(indexId)
-    val globalAccepts = getGlobalFilter(indexId)
-    return combinePredicates(indexAccepts, globalAccepts)
+  private fun acceptsInput(indexId: ID<*, *>, fileType: FileType?): IndexedFilePredicate {
+    val hint = toHint(getInputFilter(indexId))
+    val globalHint = getGlobalHint(indexId)
+
+    val indexerHintPredicate = applyHints(hint, fileType)
+    val globalHintPredicate = applyHints(globalHint, fileType)
+
+    return andPredicates(indexerHintPredicate, globalHintPredicate)
   }
 
-  private fun getIndexerFilter(indexId: ID<*, *>) = object : IndexedFilePredicate {
-    override fun test(indexedFile: IndexedFile): Boolean {
-      val filter: FileBasedIndex.InputFilter = getInputFilter(indexId)
-      return FileBasedIndexEx.acceptsInput(filter, indexedFile)
+  private fun toHint(filter: FileBasedIndex.InputFilter): IndexingHint {
+    return (filter as? IndexingHint) ?: object : IndexingHint {
+      override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean {
+        return FileBasedIndexEx.acceptsInput(filter, file)
+      }
     }
   }
 
-  private fun getGlobalFilter(indexId: ID<*, *>) = object : IndexedFilePredicate {
-    override fun test(indexedFile: IndexedFile): Boolean {
-      return !GlobalIndexFilter.isExcludedFromIndexViaFilters(indexedFile.file, indexId, indexedFile.project)
+  private fun getGlobalHint(indexId: ID<*, *>): IndexingHint = object : IndexingHint {
+    override fun whenAllOtherHintsUnsure(file: IndexedFile): Boolean {
+      return !GlobalIndexFilter.isExcludedFromIndexViaFilters(file.file, indexId, file.project)
+    }
+  }
+
+  private fun applyHints(indexingHint: IndexingHint, fileType: FileType?): IndexedFilePredicate {
+    val hint = applyFileTypeHint(indexingHint, fileType)
+    return when (hint) {
+      ThreeState.YES -> truePredicate
+      ThreeState.NO -> falsePredicate
+      ThreeState.UNSURE -> object : IndexedFilePredicate {
+        override fun test(indexedFile: IndexedFile): Boolean = indexingHint.whenAllOtherHintsUnsure(indexedFile)
+      }
+    }
+  }
+
+  private fun applyFileTypeHint(indexingHint: IndexingHint, fileType: FileType?): ThreeState {
+    if (fileType != null && indexingHint is FileTypeIndexingHint) {
+      return indexingHint.hintAcceptFileType(fileType)
+    }
+    else {
+      return ThreeState.UNSURE
     }
   }
 
@@ -102,7 +137,7 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere
       var filteredResults = indexesForFileType[fileType]
       if (filteredResults != null) return filteredResults.getRequiredIndexes(indexedFile)
 
-      filteredResults = HintAwareIndexList(getState().getFileTypesForIndex(fileType))
+      filteredResults = HintAwareIndexList(getState().getFileTypesForIndex(fileType), fileType)
       indexesForFileType[fileType] = filteredResults
       return filteredResults.getRequiredIndexes(indexedFile)
     }
@@ -116,4 +151,8 @@ internal class RequiredIndexesEvaluator(private val registeredIndexes: Registere
       return indexesForDirectories.getRequiredIndexes(indexedFile)
     }
   }
+
+  companion object {
+    private val LOG = logger<RequiredIndexesEvaluator>()
+  }
 }