- process deleted files same way as regular ones in order to process index data invalidation during bulk indexing (IDEA-147138)

- limiting disk access with nested executor services

platform/lang-impl/src/com/intellij/util/indexing/DeletedVirtualFileStub.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2000-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileWithId;
+import com.intellij.testFramework.LightVirtualFile;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Created by Maxim.Mossienko on 11/18/2015.
+ */
+class DeletedVirtualFileStub extends LightVirtualFile implements VirtualFileWithId {
+  private final int myFileId;
+
+  DeletedVirtualFileStub(VirtualFileWithId original) {
+    setOriginalFile((VirtualFile)original);
+    myFileId = -Math.abs(original.getId());
+  }
+
+  @Override
+  public int getId() {
+    return myFileId;
+  }
+
+  @Override
+  public boolean isValid() {
+    return false;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj instanceof DeletedVirtualFileStub) {
+      return ((DeletedVirtualFileStub)obj).getId() == myFileId;
+    }
+    return super.equals(obj);
+  }
+
+  @Override
+  public int hashCode() {
+    return myFileId;
+  }
+
+  @Override
+  public String toString() {
+    return "invalid:" + getOriginalFile().toString();
+  }
+
+  @NotNull
+  @Override
+  public String getUrl() {
+    return "invalid:" + super.getUrl();
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java

@@ -86,7 +86,6 @@ import java.io.*;
 import java.lang.ref.SoftReference;
 import java.lang.ref.WeakReference;
 import java.util.*;
-import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -674,7 +673,11 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     finally {
       LOG.info("START INDEX SHUTDOWN");
       try {
-        myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
+        for(VirtualFile file:myChangedFilesCollector.getAllFilesToUpdate()) {
+          if (!file.isValid()) {
+            removeDataFromIndicesForFile(file);
+          }
+        }
         IndexingStamp.flushCaches();
 
         for (ID<?, ?> indexId : myIndices.keySet()) {
@@ -696,6 +699,19 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     }
   }
 
+  private void removeDataFromIndicesForFile(VirtualFile file) {
+    final int fileId = Math.abs(getIdMaskingNonIdBasedFile(file));
+    final List<ID<?, ?>> states = IndexingStamp.getNontrivialFileIndexedStates(fileId);
+    if (!states.isEmpty()) {
+      ProgressManager.getInstance().executeNonCancelableSection(new Runnable() {
+        @Override
+        public void run() {
+          myChangedFilesCollector.removeFileDataFromIndices(states, fileId);
+        }
+      });
+    }
+  }
+
   private void flushAllIndices(final long modCount) {
     if (HeavyProcessLatch.INSTANCE.isRunning()) {
       return;
@@ -825,7 +841,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     myReentrancyGuard.set(Boolean.TRUE);
 
     try {
-      myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
       if (isUpToDateCheckEnabled()) {
         try {
           if (ourRebuildStatus.get(indexId).get() != OK) {
@@ -1585,10 +1600,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     return pair.getSecond();
   }
 
-  public int getNumberOfPendingInvalidations() {
-    return myChangedFilesCollector.getNumberOfPendingInvalidations();
-  }
-
   public int getChangedFileCount() {
     return myChangedFilesCollector.getAllFilesToUpdate().size();
   }
@@ -1598,6 +1609,9 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     return ContainerUtil.findAll(myChangedFilesCollector.getAllFilesToUpdate(), new Condition<VirtualFile>() {
       @Override
       public boolean value(VirtualFile virtualFile) {
+        if (virtualFile instanceof DeletedVirtualFileStub) {
+          return true;
+        }
         for (IndexableFileSet set : myIndexableSets) {
           final Project proj = myIndexableSetToProjectMap.get(set);
           if (proj != null && !proj.equals(project)) {
@@ -1617,7 +1631,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
   }
 
   void processRefreshedFile(@NotNull Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) {
-    myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
     myChangedFilesCollector.processFileImpl(project, fileContent); // ProcessCanceledException will cause re-adding the file to processing list
   }
 
@@ -1627,12 +1640,15 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     // in this case we consider that current indexing (out of roots backed CacheUpdater) will cover its content
     // todo this assumption isn't correct for vfs events happened between content loading and indexing itself
     // proper fix will when events handling will be out of direct execution by EDT
-    doIndexFileContent(project, content);
+    if (file.isValid()) {
+      doIndexFileContent(project, content);
+    } else {
+      removeDataFromIndicesForFile(file);
+    }
     myChangedFilesCollector.myFilesToUpdate.remove(file);
   }
 
   private void doIndexFileContent(@Nullable Project project, @NotNull com.intellij.ide.caches.FileContent content) {
-    myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
     final VirtualFile file = content.getVirtualFile();
 
     FileType fileType = file.getFileType();
@@ -1736,7 +1752,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
 
   static final Key<Boolean> ourPhysicalContentKey = Key.create("physical.content.flag");
 
-  private void updateSingleIndex(@NotNull ID<?, ?> indexId, final int inputId, @Nullable FileContent currentFC)
+  void updateSingleIndex(@NotNull ID<?, ?> indexId, final int inputId, @Nullable FileContent currentFC)
     throws StorageException {
     if (ourRebuildStatus.get(indexId).get() == REQUIRES_REBUILD && !myIsUnitTestMode) {
       return; // the index is scheduled for rebuild, no need to update
@@ -1878,7 +1894,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
 
   private final class ChangedFilesCollector extends VirtualFileAdapter implements BulkFileListener {
     private final Set<VirtualFile> myFilesToUpdate = ContainerUtil.newConcurrentSet();
-    private final Queue<InvalidationTask> myFutureInvalidations = new ConcurrentLinkedQueue<InvalidationTask>();
 
     private final ManagingFS myManagingFS = ManagingFS.getInstance();
 
@@ -2095,19 +2110,13 @@ public class FileBasedIndexImpl extends FileBasedIndex {
         }
       }
       else if (!fileIndexedStatesToUpdate.isEmpty()) { // file was removed, its data should be (lazily) wiped for every index
-        final Collection<ID<?, ?>> finalFileIndexedStatesToUpdate = fileIndexedStatesToUpdate;
+        myFilesToUpdate.add(new DeletedVirtualFileStub((VirtualFileWithId)file));
-        myFutureInvalidations.offer(new InvalidationTask(file) {
-          @Override
-          public void run() {
-            removeFileDataFromIndices(finalFileIndexedStatesToUpdate, fileId);
-          }
-        });
       }
 
       IndexingStamp.flushCache(fileId);
     }
 
-    private void removeFileDataFromIndices(@NotNull Collection<ID<?, ?>> affectedIndices, int inputId) {
+    void removeFileDataFromIndices(@NotNull Collection<ID<?, ?>> affectedIndices, int inputId) {
       Throwable unexpectedError = null;
       for (ID<?, ?> indexId : affectedIndices) {
         try {
@@ -2133,59 +2142,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
       }
     }
 
-    public int getNumberOfPendingInvalidations() {
-      return myFutureInvalidations.size();
-    }
-
-    public void ensureAllInvalidateTasksCompleted() {
-      ensureAllInvalidateTasksCompleted(false);
-    }
-
-    public void tryToEnsureAllInvalidateTasksCompleted() {
-      ensureAllInvalidateTasksCompleted(true);
-    }
-
-    private void ensureAllInvalidateTasksCompleted(boolean doCheckCancelledBetweenInvalidations) {
-      final int size = getNumberOfPendingInvalidations();
-      if (size == 0) {
-        return;
-      }
-
-      if (doCheckCancelledBetweenInvalidations) {
-        while (true) {
-          InvalidationTask task = myFutureInvalidations.poll();
-
-          if (task == null) {
-            break;
-          }
-
-          ProgressManager.getInstance().executeNonCancelableSection(task);
-          ProgressManager.checkCanceled();
-        }
-      }
-      else {
-        ProgressManager.getInstance().executeNonCancelableSection(
-          new Runnable() {
-            @Override
-            public void run() {
-              final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
-              indicator.setText("");
-              int count = 0;
-              while (true) {
-                InvalidationTask task = myFutureInvalidations.poll();
-
-                if (task == null) {
-                  break;
-                }
-                indicator.setFraction((double)count++ / size);
-                task.run();
-              }
-            }
-          }
-        );
-      }
-    }
-
     private void iterateIndexableFiles(@NotNull final VirtualFile file, @NotNull final Processor<VirtualFile> processor) {
       if (file.isDirectory()) {
         final ContentIterator iterator = new ContentIterator() {
@@ -2217,9 +2173,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     private final VirtualFileUpdateTask myForceUpdateTask = new VirtualFileUpdateTask();
     private final AtomicInteger myForceUpdateRequests = new AtomicInteger();
 
-    private void forceUpdate(@Nullable Project project, @Nullable final GlobalSearchScope filter, @Nullable final VirtualFile restrictedTo) {
+    void forceUpdate(@Nullable Project project, @Nullable final GlobalSearchScope filter, @Nullable final VirtualFile restrictedTo) {
-      myChangedFilesCollector.tryToEnsureAllInvalidateTasksCompleted();
-
       Collection<VirtualFile> allFilesToUpdate = getAllFilesToUpdate();
 
       if (!allFilesToUpdate.isEmpty()) {
@@ -2242,21 +2196,18 @@ public class FileBasedIndexImpl extends FileBasedIndex {
     private boolean processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) {
       final VirtualFile file = fileContent.getVirtualFile();
       if (myFilesToUpdate.contains(file)) {
-        if (file.isValid()) {
+        try {
-          int fileId = getIdMaskingNonIdBasedFile(file);
+          if (!file.isValid() || isTooLarge(file)) {
-          try {
+            removeDataFromIndicesForFile(file);
-            if (isTooLarge(file)) {
-              List<ID<?, ?>> nontrivialFileIndexedStates = IndexingStamp.getNontrivialFileIndexedStates(fileId);
-              removeFileDataFromIndices(ContainerUtil.intersection(nontrivialFileIndexedStates, myRequiringContentIndices), Math.abs(fileId));
-            }
-            else {
-              doIndexFileContent(project, fileContent);
-            }
           }
-          finally {
+          else {
-            IndexingStamp.flushCache(file);
+            doIndexFileContent(project, fileContent);
           }
         }
+        finally {
+          IndexingStamp.flushCache(file);
+        }
+
         myFilesToUpdate.remove(file);
         return true;
       }
@@ -2509,16 +2460,19 @@ public class FileBasedIndexImpl extends FileBasedIndex {
   @Override
   public void removeIndexableSet(@NotNull IndexableFileSet set) {
     if (!myIndexableSetToProjectMap.containsKey(set)) return;
-    myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
-    IndexingStamp.flushCaches();
     myIndexableSets.remove(set);
     myIndexableSetToProjectMap.remove(set);
 
     for (VirtualFile file : myChangedFilesCollector.getAllFilesToUpdate()) {
-      if (getIndexableSetForFile(file) == null) {
+      if (!file.isValid()) {
+        removeDataFromIndicesForFile(file);
+        myChangedFilesCollector.myFilesToUpdate.remove(file);
+      } else if (getIndexableSetForFile(file) == null) { // todo remove data from indices for removed
         myChangedFilesCollector.myFilesToUpdate.remove(file);
       }
     }
+
+    IndexingStamp.flushCaches();
   }
 
   @Override

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java

@@ -142,7 +142,7 @@ public class FileBasedIndexProjectHandler extends AbstractProjectComponent imple
     }
     final FileBasedIndexImpl index = (FileBasedIndexImpl)i;
 
-    if (index.getChangedFileCount() + index.getNumberOfPendingInvalidations() < 20) {
+    if (index.getChangedFileCount() < 20) {
       return null;
     }
 

platform/lang-impl/src/com/intellij/util/indexing/ProjectFilesCondition.java

@@ -42,12 +42,17 @@ class ProjectFilesCondition implements Condition<VirtualFile> {
 
   @Override
   public boolean value(VirtualFile file) {
-    if (myIndexableFilesFilter != null && !myIndexableFilesFilter.containsFileId(((VirtualFileWithId)file).getId())) {
+    int fileId = ((VirtualFileWithId)file).getId();
+    if (myIndexableFilesFilter != null && fileId > 0 && !myIndexableFilesFilter.containsFileId(fileId)) {
       if (myFilesFromOtherProjects >= MAX_FILES_TO_UPDATE_FROM_OTHER_PROJECT) return false;
       ++myFilesFromOtherProjects;
       return true;
     }
 
+    if (fileId < 0 && file instanceof DeletedVirtualFileStub) {
+      //file = ((FileBasedIndexImpl.MyLightVirtualFile)file).getOriginalFile();
+      return true;
+    }
     if (FileBasedIndexImpl.belongsToScope(file, myRestrictedTo, myFilter)) return true;
 
     if (myFilesFromOtherProjects < MAX_FILES_TO_UPDATE_FROM_OTHER_PROJECT) {

platform/platform-impl/src/com/intellij/openapi/project/CacheUpdateRunner.java

@@ -31,8 +31,10 @@ import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.registry.Registry;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.Consumer;
+import com.intellij.util.concurrency.BoundedTaskExecutor;
 import gnu.trove.THashSet;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.ide.PooledThreadExecutor;
 
 import java.util.Collection;
 import java.util.Set;
@@ -92,6 +94,11 @@ public class CacheUpdateRunner {
     }
   }
 
+  private static final BoundedTaskExecutor ourCacheUpdateExecutor = new BoundedTaskExecutor(
+    PooledThreadExecutor.INSTANCE,
+    indexingThreadCount()
+  );
+
   private static boolean processSomeFilesWhileUserIsInactive(@NotNull FileContentQueue queue,
                                                              @NotNull Consumer<VirtualFile> progressUpdater,
                                                              final boolean processInReadAction,
@@ -131,7 +138,7 @@ public class CacheUpdateRunner {
           AtomicBoolean ref = new AtomicBoolean();
           finishedRefs[i] = ref;
           Runnable process = new MyRunnable(innerIndicator, queue, ref, progressUpdater, processInReadAction, project, fileProcessor);
-          futures[i] = ApplicationManager.getApplication().executeOnPooledThread(process);
+          futures[i] = ourCacheUpdateExecutor.submit(process);
         }
         isFinished.set(waitForAll(finishedRefs, futures));
       }
@@ -221,7 +228,7 @@ public class CacheUpdateRunner {
                 final VirtualFile file = fileContent.getVirtualFile();
                 try {
                   myProgressUpdater.consume(file);
-                  if (file.isValid() && !file.isDirectory() && !Boolean.TRUE.equals(file.getUserData(FAILED_TO_INDEX))) {
+                  if (!file.isDirectory() && !Boolean.TRUE.equals(file.getUserData(FAILED_TO_INDEX))) {
                     myProcessor.consume(fileContent);
                   }
                 }