Indexes roots collecting re-implemented.

Instead of collecting Runnables that iterate over all indexable files let's expose roots from where to start iteration. This will allow to prioritize roots by some predicate.

This commit was reviewed in IDEA-CR-57771

GitOrigin-RevId: 8162e759fb27b1bdf64fc24e1a13d454b9a28d5d

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexEx.java

@@ -1,10 +1,15 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.util.indexing;
 
+import com.intellij.ide.lightEdit.LightEditUtil;
+import com.intellij.openapi.application.ReadAction;
+import com.intellij.openapi.module.Module;
+import com.intellij.openapi.module.ModuleManager;
+import com.intellij.openapi.progress.EmptyProgressIndicator;
 import com.intellij.openapi.progress.ProgressIndicator;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.roots.ContentIterator;
+import com.intellij.openapi.roots.*;
 import com.intellij.openapi.roots.impl.PushedFilePropertiesUpdaterImpl;
 import com.intellij.openapi.util.Condition;
 import com.intellij.openapi.vfs.VirtualFile;
@@ -16,6 +21,7 @@ import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.*;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.impl.InvertedIndexValueIterator;
+import com.intellij.util.indexing.roots.*;
 import gnu.trove.THashSet;
 import gnu.trove.TIntHashSet;
 import org.jetbrains.annotations.ApiStatus;
@@ -315,21 +321,82 @@ public abstract class FileBasedIndexEx extends FileBasedIndex {
   }
 
   @Override
-  public void iterateIndexableFilesConcurrently(@NotNull ContentIterator processor, @NotNull Project project, @NotNull ProgressIndicator indicator) {
-    PushedFilePropertiesUpdaterImpl.invokeConcurrentlyIfPossible(collectScanRootRunnables(processor, project, indicator));
+  public void iterateIndexableFilesConcurrently(@NotNull ContentIterator processor,
+                                                @NotNull Project project,
+                                                @NotNull ProgressIndicator indicator) {
+    List<Runnable> tasks = collectIndexableFilesIterateTasks(processor, project, indicator);
+    if (!tasks.isEmpty()) {
+      PushedFilePropertiesUpdaterImpl.invokeConcurrentlyIfPossible(tasks);
+    }
   }
 
   @Override
-  public void iterateIndexableFiles(@NotNull ContentIterator processor, @NotNull Project project, ProgressIndicator indicator) {
-    for(Runnable r: collectScanRootRunnables(processor, project, indicator)) r.run();
+  public void iterateIndexableFiles(@NotNull ContentIterator processor, @NotNull Project project, @Nullable ProgressIndicator indicator) {
+    final ProgressIndicator finalIndicator = indicator == null ? new EmptyProgressIndicator() : indicator;
+    List<Runnable> tasks = collectIndexableFilesIterateTasks(processor, project, finalIndicator);
+    for (Runnable task : tasks) {
+      task.run();
+    }
   }
 
   @NotNull
-  private static List<Runnable> collectScanRootRunnables(@NotNull final ContentIterator processor,
-                                                         @NotNull final Project project,
-                                                         ProgressIndicator indicator) {
-    FileBasedIndexScanRunnableCollector collector = FileBasedIndexScanRunnableCollector.getInstance(project);
-    return collector.collectScanRootRunnables(processor, indicator);
+  private static List<Runnable> collectIndexableFilesIterateTasks(@NotNull ContentIterator processor,
+                                                                  @NotNull Project project,
+                                                                  @NotNull ProgressIndicator indicator) {
+    if (LightEditUtil.isLightEditProject(project)) {
+      return Collections.emptyList();
+    }
+    @NotNull List<IndexableRootsProvider> providers = getIndexableRootsProvider(project);
+    ProjectFileIndex projectFileIndex = ProjectFileIndex.getInstance(project);
+    Set<VirtualFile> visitedRoots = ContainerUtil.newConcurrentSet();
+    return ContainerUtil.map(providers, provider -> () -> {
+      Set<VirtualFile> rootsToIndex = provider.getRootsToIndex();
+      for (VirtualFile root : rootsToIndex) {
+        if (visitedRoots.add(root)) {
+          FileBasedIndex.iterateRecursively(root, processor, indicator, visitedRoots, projectFileIndex);
+        }
+      }
+    });
+  }
+
+  @NotNull
+  private static List<IndexableRootsProvider> getIndexableRootsProvider(@NotNull Project project) {
+    return ReadAction.compute(() -> {
+      if (project.isDisposed()) {
+        return Collections.emptyList();
+      }
+
+      List<IndexableRootsProvider> providers = new ArrayList<>();
+      Module[] modules = ModuleManager.getInstance(project).getModules();
+      for (Module module : modules) {
+        if (module.isDisposed()) continue;
+        providers.add(new ModuleIndexableRootsProvider(module));
+
+        // iterate associated libraries
+        OrderEntry[] orderEntries = ModuleRootManager.getInstance(module).getOrderEntries();
+        for (OrderEntry orderEntry : orderEntries) {
+          if (!(orderEntry instanceof LibraryOrSdkOrderEntry) || !orderEntry.isValid()) {
+            continue;
+          }
+
+          final LibraryOrSdkOrderEntry entry = (LibraryOrSdkOrderEntry)orderEntry;
+          providers.add(new LibraryOrSdkOrderEntryIndexableRootsProvider(entry));
+        }
+      }
+
+      for (IndexableSetContributor contributor : IndexableSetContributor.EP_NAME.getExtensionList()) {
+        providers.add(new IndexableSetContributorRootsProvider(contributor, project));
+      }
+
+      // iterate synthetic project libraries
+      for (AdditionalLibraryRootsProvider provider : AdditionalLibraryRootsProvider.EP_NAME.getExtensionList()) {
+        for (SyntheticLibrary library : provider.getAdditionalProjectLibraries(project)) {
+          providers.add(new SyntheticLibraryIndexableRootsProvider(library));
+        }
+      }
+
+      return providers;
+    });
   }
 
   @Nullable

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java

@@ -8,12 +8,15 @@ package com.intellij.util.indexing;
 import com.google.common.annotations.VisibleForTesting;
 import com.intellij.diagnostic.PerformanceWatcher;
 import com.intellij.ide.IdeBundle;
+import com.intellij.ide.lightEdit.LightEditUtil;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.components.Service;
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.fileTypes.FileTypeManager;
 import com.intellij.openapi.progress.ProgressIndicator;
 import com.intellij.openapi.project.*;
 import com.intellij.openapi.roots.ContentIterator;
+import com.intellij.openapi.roots.ProjectFileIndex;
 import com.intellij.openapi.roots.impl.PushedFilePropertiesUpdater;
 import com.intellij.openapi.startup.StartupActivity;
 import com.intellij.openapi.util.Disposer;
@@ -32,13 +35,14 @@ import java.util.Collection;
 @Service
 public final class FileBasedIndexProjectHandler implements IndexableFileSet {
   private static final Logger LOG = Logger.getInstance(FileBasedIndexProjectHandler.class);
-
-  private final FileBasedIndexScanRunnableCollector myCollector;
+  private final Project myProject;
+  private final @NotNull ProjectFileIndex myProjectFileIndex;
 
   private boolean isRemoved;
 
   private FileBasedIndexProjectHandler(@NotNull Project project) {
-    myCollector = FileBasedIndexScanRunnableCollector.getInstance(project);
+    myProject = project;
+    myProjectFileIndex = ProjectFileIndex.getInstance(myProject);
   }
 
   static final class FileBasedIndexProjectHandlerStartupActivity implements StartupActivity {
@@ -86,7 +90,13 @@ public final class FileBasedIndexProjectHandler implements IndexableFileSet {
 
   @Override
   public boolean isInSet(@NotNull final VirtualFile file) {
-    return myCollector.shouldCollect(file);
+    if (LightEditUtil.isLightEditProject(myProject)) {
+      return false;
+    }
+    if (myProjectFileIndex.isInContent(file) || myProjectFileIndex.isInLibrary(file)) {
+      return !FileTypeManager.getInstance().isFileIgnored(file);
+    }
+    return false;
   }
 
   @Override

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexScanRunnableCollector.java

@@ -1,23 +1,42 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.util.indexing;
 
-import com.intellij.openapi.components.ServiceManager;
-import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.ide.lightEdit.LightEditUtil;
+import com.intellij.openapi.fileTypes.FileTypeManager;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.roots.ContentIterator;
+import com.intellij.openapi.roots.ProjectFileIndex;
 import com.intellij.openapi.vfs.VirtualFile;
 import org.jetbrains.annotations.NotNull;
 
-import java.util.List;
+/**
+ * @deprecated Use {@link FileBasedIndexRootsCollector} instead.
+ */
+@Deprecated
+public final class FileBasedIndexScanRunnableCollector {
 
-public abstract class FileBasedIndexScanRunnableCollector {
-  public static FileBasedIndexScanRunnableCollector getInstance(@NotNull Project project) {
-    return ServiceManager.getService(project, FileBasedIndexScanRunnableCollector.class);
+  private final Project myProject;
+  private final @NotNull ProjectFileIndex myProjectFileIndex;
+
+  public FileBasedIndexScanRunnableCollector(Project project) {
+    myProject = project;
+    myProjectFileIndex = ProjectFileIndex.getInstance(project);
   }
 
-  // Returns true if file should be indexed
-  public abstract boolean shouldCollect(@NotNull final VirtualFile file);
+  public static FileBasedIndexScanRunnableCollector getInstance(@NotNull Project project) {
+    return new FileBasedIndexScanRunnableCollector(project);
+  }
 
-  // Collect all roots for indexing
-  public abstract List<Runnable> collectScanRootRunnables(@NotNull final ContentIterator processor, final ProgressIndicator indicator);
+  /**
+   * @deprecated Use ProjectFileIndex directly.
+   */
+  @Deprecated
+  public final boolean shouldCollect(@NotNull final VirtualFile file) {
+    if (LightEditUtil.isLightEditProject(myProject)) {
+      return false;
+    }
+    if (myProjectFileIndex.isInContent(file) || myProjectFileIndex.isInLibrary(file)) {
+      return !FileTypeManager.getInstance().isFileIgnored(file);
+    }
+    return false;
+  }
 }

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexScanRunnableCollectorImpl.java

@@ -1,119 +0,0 @@
-// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.intellij.util.indexing;
-
-import com.intellij.ide.lightEdit.LightEditUtil;
-import com.intellij.openapi.application.ReadAction;
-import com.intellij.openapi.fileTypes.FileTypeManager;
-import com.intellij.openapi.module.Module;
-import com.intellij.openapi.module.ModuleManager;
-import com.intellij.openapi.progress.ProgressIndicator;
-import com.intellij.openapi.project.Project;
-import com.intellij.openapi.roots.*;
-import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.util.containers.ContainerUtil;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.*;
-
-final class FileBasedIndexScanRunnableCollectorImpl extends FileBasedIndexScanRunnableCollector {
-  private final Project myProject;
-  private final ProjectFileIndex myProjectFileIndex;
-  private final boolean myDisabled;
-
-  FileBasedIndexScanRunnableCollectorImpl(@NotNull Project project) {
-    myProject = project;
-    myProjectFileIndex = ProjectFileIndex.getInstance(myProject);
-    myDisabled = LightEditUtil.isLightEditProject(myProject);
-  }
-
-  @Override
-  public boolean shouldCollect(@NotNull VirtualFile file) {
-    if (myDisabled) return false;
-    if (myProjectFileIndex.isInContent(file) || myProjectFileIndex.isInLibrary(file)) {
-      return !FileTypeManager.getInstance().isFileIgnored(file);
-    }
-    return false;
-  }
-
-  @Override
-  public List<Runnable> collectScanRootRunnables(@NotNull ContentIterator processor, ProgressIndicator indicator) {
-    if (myDisabled) {
-      return Collections.emptyList();
-    }
-    return ReadAction.compute(() -> {
-      if (myProject.isDisposed()) {
-        return Collections.emptyList();
-      }
-
-      List<Runnable> tasks = new ArrayList<>();
-      final Set<VirtualFile> visitedRoots = ContainerUtil.newConcurrentSet();
-
-      tasks.add(() -> myProjectFileIndex.iterateContent(processor, file -> !file.isDirectory() || visitedRoots.add(file)));
-
-      Set<VirtualFile> contributedRoots = new LinkedHashSet<>();
-      for (IndexableSetContributor contributor : IndexableSetContributor.EP_NAME.getExtensionList()) {
-        //important not to depend on project here, to support per-project background reindex
-        // each client gives a project to FileBasedIndex
-        if (myProject.isDisposed()) {
-          return tasks;
-        }
-        contributedRoots.addAll(IndexableSetContributor.getRootsToIndex(contributor));
-        contributedRoots.addAll(IndexableSetContributor.getProjectRootsToIndex(contributor, myProject));
-      }
-      for (VirtualFile root : contributedRoots) {
-        // do not try to visit under-content-roots because the first task took care of that already
-        if (!myProjectFileIndex.isInContent(root) && visitedRoots.add(root)) {
-          tasks.add(() -> {
-            if (myProject.isDisposed() || !root.isValid()) return;
-            FileBasedIndex.iterateRecursively(root, processor, indicator, visitedRoots, null);
-          });
-        }
-      }
-
-      // iterate synthetic project libraries
-      for (AdditionalLibraryRootsProvider provider : AdditionalLibraryRootsProvider.EP_NAME.getExtensionList()) {
-        if (myProject.isDisposed()) {
-          return tasks;
-        }
-        for (SyntheticLibrary library : provider.getAdditionalProjectLibraries(myProject)) {
-          for (VirtualFile root : library.getAllRoots()) {
-            // do not try to visit under-content-roots because the first task took care of that already
-            if (!myProjectFileIndex.isInContent(root) && visitedRoots.add(root)) {
-              tasks.add(() -> {
-                if (myProject.isDisposed() || !root.isValid()) return;
-                FileBasedIndex.iterateRecursively(root, processor, indicator, visitedRoots, myProjectFileIndex);
-              });
-            }
-          }
-        }
-      }
-
-      // iterate associated libraries
-      for (final Module module : ModuleManager.getInstance(myProject).getModules()) {
-        OrderEntry[] orderEntries = ModuleRootManager.getInstance(module).getOrderEntries();
-        for (OrderEntry orderEntry : orderEntries) {
-          if (!(orderEntry instanceof LibraryOrSdkOrderEntry) || !orderEntry.isValid()) {
-            continue;
-          }
-
-          final LibraryOrSdkOrderEntry entry = (LibraryOrSdkOrderEntry)orderEntry;
-          final VirtualFile[] libSources = entry.getRootFiles(OrderRootType.SOURCES);
-          final VirtualFile[] libClasses = entry.getRootFiles(OrderRootType.CLASSES);
-          for (VirtualFile[] roots : new VirtualFile[][]{libSources, libClasses}) {
-            for (final VirtualFile root : roots) {
-              // do not try to visit under-content-roots because the first task took care of that already
-              if (!myProjectFileIndex.isInContent(root) && visitedRoots.add(root)) {
-                tasks.add(() -> {
-                  if (myProject.isDisposed() || module.isDisposed() || !root.isValid()) return;
-                  FileBasedIndex.iterateRecursively(root, processor, indicator, visitedRoots, myProjectFileIndex);
-                });
-              }
-            }
-          }
-        }
-      }
-
-      return tasks;
-    });
-  }
-}

platform/lang-impl/src/com/intellij/util/indexing/roots/IndexableRootsProvider.kt

@@ -0,0 +1,12 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.roots
+
+import com.intellij.openapi.vfs.VirtualFile
+import org.jetbrains.annotations.ApiStatus
+
+@ApiStatus.Internal
+interface IndexableRootsProvider {
+  val presentableName: String
+
+  fun getRootsToIndex(): Set<VirtualFile>
+}

platform/lang-impl/src/com/intellij/util/indexing/roots/IndexableSetContributorRootsProvider.kt

@@ -0,0 +1,24 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.roots
+
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.util.indexing.IndexableSetContributor
+
+data class IndexableSetContributorRootsProvider(
+  val indexableSetContributor: IndexableSetContributor,
+  val project: Project
+) : IndexableRootsProvider {
+
+  override val presentableName
+    get() = "Additional roots"
+
+  override fun getRootsToIndex() = runReadAction {
+    val roots = linkedSetOf<VirtualFile>()
+    roots += indexableSetContributor.getAdditionalProjectRootsToIndex(project)
+    roots += indexableSetContributor.additionalRootsToIndex
+    roots
+  }
+
+}

platform/lang-impl/src/com/intellij/util/indexing/roots/LibraryOrSdkOrderEntryIndexableRootsProvider.kt

@@ -0,0 +1,19 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.roots
+
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.roots.LibraryOrSdkOrderEntry
+import com.intellij.openapi.roots.OrderRootType
+import com.intellij.openapi.vfs.VirtualFile
+
+data class LibraryOrSdkOrderEntryIndexableRootsProvider(val orderEntry: LibraryOrSdkOrderEntry) : IndexableRootsProvider {
+  override val presentableName
+    get() = "Roots of ${orderEntry.presentableName}"
+
+  override fun getRootsToIndex() = runReadAction {
+    val roots = linkedSetOf<VirtualFile>()
+    roots += orderEntry.getRootFiles(OrderRootType.SOURCES)
+    roots += orderEntry.getRootFiles(OrderRootType.CLASSES)
+    roots
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/roots/ModuleIndexableRootsProvider.kt

@@ -0,0 +1,17 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.roots
+
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.module.Module
+import com.intellij.openapi.roots.ModuleRootManager
+import com.intellij.openapi.vfs.VirtualFile
+
+data class ModuleIndexableRootsProvider(val module: Module) : IndexableRootsProvider {
+  override val presentableName
+    get() = "Roots of module ${module.name}"
+
+  override fun getRootsToIndex(): Set<VirtualFile> = runReadAction {
+    ModuleRootManager.getInstance(module).fileIndex.moduleRootsToIterate
+  }
+
+}

platform/lang-impl/src/com/intellij/util/indexing/roots/SyntheticLibraryIndexableRootsProvider.kt

@@ -0,0 +1,15 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.roots
+
+import com.intellij.navigation.ItemPresentation
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.roots.SyntheticLibrary
+
+data class SyntheticLibraryIndexableRootsProvider(val syntheticLibrary: SyntheticLibrary) : IndexableRootsProvider {
+  override val presentableName
+    get() = "Roots of " + if (syntheticLibrary is ItemPresentation) syntheticLibrary.presentableText else "synthetic library"
+
+  override fun getRootsToIndex() = runReadAction {
+    syntheticLibrary.allRoots.toSet()
+  }
+}

platform/platform-resources/src/META-INF/LangExtensions.xml

@@ -115,9 +115,6 @@
     <projectService serviceInterface="com.intellij.psi.PsiDocumentManager"
                     serviceImplementation="com.intellij.psi.impl.PsiDocumentManagerImpl" preload="await"/>
 
-    <projectService serviceInterface="com.intellij.util.indexing.FileBasedIndexScanRunnableCollector"
-                    serviceImplementation="com.intellij.util.indexing.FileBasedIndexScanRunnableCollectorImpl"/>
-
     <projectService serviceInterface="com.intellij.pom.references.PomService"
                     serviceImplementation="com.intellij.pom.references.PomServiceImpl"/>
 

platform/projectModel-api/src/com/intellij/openapi/roots/ModuleFileIndex.java

@@ -16,11 +16,13 @@
 package com.intellij.openapi.roots;
 
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileFilter;
 import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.List;
+import java.util.Set;
 
 /**
  * Provides information about files contained in a module. Should be used from a read action.
@@ -50,4 +52,13 @@ public interface ModuleFileIndex extends FileIndex {
    * @return the list of order entries to which the file or directory belongs.
    */
   @NotNull List<OrderEntry> getOrderEntriesForFile(@NotNull VirtualFile fileOrDir);
+
+  /**
+   * Returns content roots of the module (skipping excluded and ignored files and directories)
+   * that will be recursively iterated when one of the iterate methods is called.
+   *
+   * @see #iterateContent(ContentIterator)
+   * @see #iterateContent(ContentIterator, VirtualFileFilter)
+   */
+  @NotNull Set<VirtualFile> getModuleRootsToIterate();
 }

platform/projectModel-impl/src/com/intellij/openapi/roots/impl/ModuleFileIndexImpl.java

@@ -35,8 +35,9 @@ public class ModuleFileIndexImpl extends FileIndexBase implements ModuleFileInde
   }
 
 
+  @Override
   @NotNull
-  Set<VirtualFile> getModuleRootsToIterate() {
+  public Set<VirtualFile> getModuleRootsToIterate() {
     return ReadAction.compute(() -> {
       if (myModule.isDisposed()) return Collections.emptySet();
       Set<VirtualFile> result = new LinkedHashSet<>();

platform/projectModel-impl/src/com/intellij/openapi/roots/impl/ProjectFileIndexImpl.java

@@ -59,7 +59,7 @@ public class ProjectFileIndexImpl extends FileIndexBase implements ProjectFileIn
     return ReadAction.compute(() -> {
       if (module.isDisposed()) return Collections.emptySet();
 
-      ModuleFileIndexImpl moduleFileIndex = (ModuleFileIndexImpl)ModuleRootManager.getInstance(module).getFileIndex();
+      ModuleFileIndex moduleFileIndex = ModuleRootManager.getInstance(module).getFileIndex();
       Set<VirtualFile> result = moduleFileIndex.getModuleRootsToIterate();
 
       for (Iterator<VirtualFile> iterator = result.iterator(); iterator.hasNext(); ) {