From 814c5522dbadbfa64eddb88bfbf04bcad080e8b2 Mon Sep 17 00:00:00 2001
From: peter <peter@jetbrains.com>
Date: Fri, 5 Jan 2018 16:31:52 +0100
Subject: [PATCH] IDEA-123287 Multiple "Updating indices" bg tasks after
 external file system changes

---
 .../util/indexing/FileBasedIndexImpl.java     | 13 +++----
 .../FileBasedIndexProjectHandler.java         | 35 ++++++++++++++++---
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
index bfd3ad540095..bb9955e04a2b 100644
--- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
+++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
@@ -90,7 +90,6 @@ import gnu.trove.THashMap;
 import gnu.trove.THashSet;
 import gnu.trove.TIntArrayList;
 import gnu.trove.TIntHashSet;
-import one.util.streamex.StreamEx;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;
@@ -99,7 +98,10 @@ import java.io.*;
 import java.lang.ref.SoftReference;
 import java.lang.ref.WeakReference;
 import java.util.*;
-import java.util.concurrent.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -108,7 +110,6 @@ import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Consumer;
-import java.util.stream.Stream;
 
 /**
  * @author Eugene Zhuravlev
@@ -1471,12 +1472,6 @@ public class FileBasedIndexImpl extends FileBasedIndex implements BaseComponent,
     return myChangedFilesCollector.myVfsEventsMerger.getApproximateChangesCount() + myChangedFilesCollector.myFilesToUpdate.size();
   }
 
-  String dumpSomeChangedFiles() {
-    Stream<VirtualFile> events = myChangedFilesCollector.myVfsEventsMerger.getChangedFiles();
-    Stream<VirtualFile> files = myChangedFilesCollector.myFilesToUpdate.values().stream();
-    return StreamEx.of(events).append(files).limit(20).map(VirtualFile::getPath).joining(", ");
-  }
-
   @NotNull
   Collection<VirtualFile> getFilesToUpdate(final Project project) {
     return ContainerUtil.findAll(myChangedFilesCollector.getAllFilesToUpdate(), virtualFile -> {
diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java
index a022c8a882b9..d31eecde95f6 100644
--- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java
+++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java
@@ -35,9 +35,11 @@ import com.intellij.openapi.util.registry.Registry;
 import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileVisitor;
+import one.util.streamex.StreamEx;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.ArrayList;
 import java.util.Collection;
 
 public class FileBasedIndexProjectHandler implements IndexableFileSet, Disposable {
@@ -125,11 +127,14 @@ public class FileBasedIndexProjectHandler implements IndexableFileSet, Disposabl
       return null;
     }
 
-    final FileBasedIndexImpl index = (FileBasedIndexImpl)i;
-    if (index.getChangedFileCount() < ourMinFilesToStartDumMode) {
-      if (index.getChangedFilesSize() < ourMinFilesSizeToStartDumMode) return null;
+    FileBasedIndexImpl index = (FileBasedIndexImpl)i;
+    if (index.getChangedFileCount() < ourMinFilesToStartDumMode && index.getChangedFilesSize() < ourMinFilesSizeToStartDumMode) {
+      return null;
     }
 
+    Collection<VirtualFile> files = getFilesToReindexIfTooMany(project, index);
+    if (files == null) return null;
+
     return new DumbModeTask(project.getComponent(FileBasedIndexProjectHandler.class)) {
       @Override
       public void performInDumbMode(@NotNull ProgressIndicator indicator) {
@@ -150,11 +155,33 @@ public class FileBasedIndexProjectHandler implements IndexableFileSet, Disposabl
 
       @Override
       public String toString() {
-        return getClass().getName() + "[" + index.dumpSomeChangedFiles() + "]";
+        return getClass().getName() + "[" + StreamEx.of(files).limit(20).map(VirtualFile::getPath).joining(", ") + "]";
       }
     };
   }
 
+  @Nullable
+  private static Collection<VirtualFile> getFilesToReindexIfTooMany(Project project, FileBasedIndexImpl index) {
+    long start = System.currentTimeMillis();
+
+    Collection<VirtualFile> files = new ArrayList<>();
+    try {
+      files = index.getFilesToUpdate(project);
+      if (files.size() < ourMinFilesToStartDumMode &&
+          files.stream().mapToLong(VirtualFile::getLength).sum() < ourMinFilesSizeToStartDumMode) {
+        return null;
+      }
+    }
+    finally {
+      long took = System.currentTimeMillis() - start;
+      if (took > 10) {
+        LOG.info("Synchronous checking for files to reindex took " + took + "ms, " +
+                 "found " + files.size() + " of total " + index.getChangedFileCount() + " changed");
+      }
+    }
+    return files;
+  }
+
   private static void reindexRefreshedFiles(ProgressIndicator indicator,
                                             Collection<VirtualFile> files,
                                             final Project project,