MapReduceIndex moved to [util]

java/java-tests/testSrc/com/intellij/index/IndexTest.groovy

@@ -154,7 +154,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase {
     final File storageFile = FileUtil.createTempFile("index_test", "storage")
     final File metaIndexFile = FileUtil.createTempFile("index_test_inputs", "storage")
     PersistentHashMap<Integer, Collection<String>>  index = createMetaIndex(metaIndexFile)
-    final MapIndexStorage indexStorage = new MapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024)
+    final VfsAwareMapIndexStorage indexStorage = new VfsAwareMapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024)
     return new StringIndex(testName, indexStorage, index)
   }
   

java/java-tests/testSrc/com/intellij/index/StringIndex.java

@@ -17,12 +17,16 @@ package com.intellij.index;
 
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.impl.IndexStorage;
+import com.intellij.util.indexing.impl.MapBasedForwardIndex;
+import com.intellij.util.indexing.impl.MapReduceIndex;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.EnumeratorStringDescriptor;
 import com.intellij.util.io.KeyDescriptor;
 import com.intellij.util.io.PersistentHashMap;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -39,11 +43,12 @@ public class StringIndex {
   
   public StringIndex(String testName, final IndexStorage<String, String> storage, final PersistentHashMap<Integer, Collection<String>> inputIndex)
     throws IOException {
-    myIndex = new MapReduceIndex<String, String, PathContentPair>(new IndexExtension<String, String, PathContentPair>() {
+    ID<String, String> id = ID.create(testName + "string_index");
+    IndexExtension<String, String, PathContentPair> extension = new IndexExtension<String, String, PathContentPair>() {
       @NotNull
       @Override
       public ID<String, String> getName() {
-        return new ID<String, String>(testName + "string_index") {};
+        return id;
       }
 
       @NotNull
@@ -68,10 +73,19 @@ public class StringIndex {
       public int getVersion() {
         return 0;
       }
-    }, storage) {
-      protected PersistentHashMap<Integer, Collection<String>> createInputsIndex() throws IOException {
+    };
+    myIndex = new VfsAwareMapReduceIndex<String, String, PathContentPair>(extension, storage, new MapBasedForwardIndex<String, String>(extension) {
+      @NotNull
+      @Override
+      public PersistentHashMap<Integer, Collection<String>> createMap() throws IOException {
         return inputIndex;
       }
+    }) {
+      @Override
+      public void requestRebuild(@NotNull Exception ex) {
+        Assert.fail();
+      }
+
     };
   }
 

platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java

@@ -31,6 +31,7 @@ import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.LanguageFileType;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
+import com.intellij.openapi.util.ThrowableComputable;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
@@ -49,6 +50,7 @@ import com.intellij.util.Processors;
 import com.intellij.util.SmartList;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.impl.*;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.DataInputOutputUtil;
 import com.intellij.util.io.KeyDescriptor;
@@ -134,7 +136,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
 
     for (int attempt = 0; attempt < 2; attempt++) {
       try {
-        final MapIndexStorage<K, StubIdList> storage = new MapIndexStorage<>(
+        final VfsAwareMapIndexStorage<K, StubIdList> storage = new VfsAwareMapIndexStorage<>(
           IndexInfrastructure.getStorageFile(indexKey),
           extension.getKeyDescriptor(),
           StubIdExternalizer.INSTANCE,
@@ -503,14 +505,14 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
 
   public void setDataBufferingEnabled(final boolean enabled) {
     for (UpdatableIndex index : getAsyncState().myIndices.values()) {
-      final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
+      final IndexStorage indexStorage = ((VfsAwareMapReduceIndex)index).getStorage();
       ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
     }
   }
 
   public void cleanupMemoryStorage() {
     for (UpdatableIndex index : getAsyncState().myIndices.values()) {
-      final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
+      final IndexStorage indexStorage = ((VfsAwareMapReduceIndex)index).getStorage();
       index.getWriteLock().lock();
       try {
         ((MemoryIndexStorage)indexStorage).clearMemoryMap();
@@ -573,32 +575,18 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
     return Collections.unmodifiableCollection(getAsyncState().myIndices.keySet());
   }
 
-  public <K> void updateIndex(@NotNull StubIndexKey key, int fileId, @NotNull final Map<K, StubIdList> oldValues, @NotNull final Map<K, StubIdList> newValues) {
+  public <K> void updateIndex(@NotNull StubIndexKey key,
+                              int fileId,
+                              @NotNull final Map<K, StubIdList> oldValues,
+                              @NotNull final Map<K, StubIdList> newValues) {
     try {
       final MyIndex<K> index = (MyIndex<K>)getAsyncState().myIndices.get(key);
-      UpdateData<K, StubIdList> updateData;
-
-      if (MapDiffUpdateData.ourDiffUpdateEnabled) {
-        updateData = new MapDiffUpdateData<K, StubIdList>(key) {
-          @Override
-          public void save(int inputId) throws IOException {
-          }
-
-          @Override
-          protected Map<K, StubIdList> getNewValue() {
-            return newValues;
-          }
-
-          @Override
-          protected Map<K, StubIdList> getCurrentValue() throws IOException {
-            return oldValues;
-          }
-        };
-      }
-      else {
-        updateData = index.new SimpleUpdateData(key, fileId, newValues, oldValues::keySet);
-      }
-      index.updateWithMap(fileId, updateData);
+      final ThrowableComputable<ForwardIndex.InputKeyIterator<K, StubIdList>, IOException>
+        oldMapGetter = () -> new MapInputKeyIterator<>(oldValues);
+      index.updateWithMap(fileId,
+                          DiffUpdateData.ourDiffUpdateEnabled
+                          ? new DiffUpdateData<>(newValues, oldMapGetter, key, null)
+                          : new SimpleUpdateData<>(newValues, oldMapGetter, key, null));
     }
     catch (StorageException e) {
       LOG.info(e);
@@ -606,7 +594,8 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
     }
   }
 
-  private static class MyIndex<K> extends MapReduceIndex<K, StubIdList, Void> {
+  private static class MyIndex<K> extends VfsAwareMapReduceIndex<K, StubIdList, Void> {
+
     public MyIndex(IndexExtension<K, StubIdList, Void> extension, IndexStorage<K, StubIdList> storage) throws IOException {
       super(extension, storage);
     }
@@ -616,6 +605,10 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
                               @NotNull UpdateData<K, StubIdList> updateData) throws StorageException {
       super.updateWithMap(inputId, updateData);
     }
+
+    public IndexExtension<K, StubIdList, Void> getExtension() {
+      return myExtension;
+    }
   }
 
   @Override

platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java

@@ -23,7 +23,7 @@ import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.LanguageFileType;
 import com.intellij.openapi.progress.ProcessCanceledException;
-import com.intellij.openapi.util.NotNullComputable;
+import com.intellij.openapi.util.ThrowableComputable;
 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.newvfs.FileAttribute;
@@ -32,8 +32,10 @@ import com.intellij.openapi.vfs.newvfs.persistent.FSRecords;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.IStubFileElementType;
 import com.intellij.util.ExceptionUtil;
+import com.intellij.util.ThrowableRunnable;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.impl.*;
 import com.intellij.util.io.*;
 import gnu.trove.THashMap;
 import org.jetbrains.annotations.NotNull;
@@ -391,7 +393,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi
     }
   }
 
-  private static class MyIndex extends MapReduceIndex<Integer, SerializedStubTree, FileContent> {
+  private static class MyIndex extends VfsAwareMapReduceIndex<Integer, SerializedStubTree, FileContent> {
     private StubIndexImpl myStubIndex;
     private final StubVersionMap myStubVersionMap = new StubVersionMap();
 
@@ -401,37 +403,32 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi
       checkNameStorage();
     }
 
+    @NotNull
     @Override
-    public void flush() throws StorageException {
-      final StubIndexImpl stubIndex = getStubIndex();
-      try {
-        stubIndex.flush();
-      }
-      finally {
-        super.flush();
-      }
+    protected UpdateData<Integer, SerializedStubTree> createUpdateData(Map<Integer, SerializedStubTree> data,
+                                                                       ThrowableComputable<ForwardIndex.InputKeyIterator<Integer, SerializedStubTree>, IOException> oldKeys,
+                                                                       ThrowableRunnable<IOException> forwardIndexUpdate) {
+      return new StubUpdatingData(data, oldKeys, forwardIndexUpdate);
     }
 
-    @Override
-    protected UpdateData<Integer, SerializedStubTree> buildUpdateData(Map<Integer, SerializedStubTree> data,
-                                                                      NotNullComputable<Collection<Integer>> oldKeysGetter,
-                                                                      int savedInputId) {
-      return new StubUpdatingData(savedInputId, data, oldKeysGetter);
-    }
-
-    class StubUpdatingData extends SimpleUpdateData {
+    static class StubUpdatingData extends SimpleUpdateData<Integer, SerializedStubTree> {
       private Collection<Integer> oldStubIndexKeys;
 
-      public StubUpdatingData(int id,
-                              @NotNull Map<Integer, SerializedStubTree> data,
-                              @NotNull NotNullComputable<Collection<Integer>> getter) {
-        super(INDEX_ID, id, data, getter);
+      public StubUpdatingData(@NotNull Map<Integer, SerializedStubTree> newData,
+                              @NotNull ThrowableComputable<ForwardIndex.InputKeyIterator<Integer, SerializedStubTree>, IOException> iterator,
+                              ThrowableRunnable<IOException> forwardIndexUpdate) {
+        super(newData, iterator, INDEX_ID, forwardIndexUpdate);
       }
 
       @Override
-      public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Integer> consumer) throws StorageException {
-        oldStubIndexKeys = oldKeysGetter.compute();
-        MapDiffUpdateData.iterateRemovedKeys(oldStubIndexKeys, inputId, consumer);
+      protected void iterateKeys(int inputId,
+                                 KeyValueUpdateProcessor<Integer, SerializedStubTree> addProcessor,
+                                 RemovedKeyProcessor<Integer> removeProcessor,
+                                 ForwardIndex.InputKeyIterator<Integer, SerializedStubTree> currentData) throws StorageException {
+        if (currentData instanceof CollectionInputKeyIterator) {
+          oldStubIndexKeys = ((CollectionInputKeyIterator<Integer, SerializedStubTree>)currentData).getCollection();
+        }
+        super.iterateKeys(inputId, addProcessor, removeProcessor, currentData);
       }
 
       public Map<StubIndexKey, Map<Object, StubIdList>> getOldStubIndicesValueMap() {
@@ -451,10 +448,20 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi
     }
 
     @Override
-    protected void updateWithMap(final int inputId,
-                                 @NotNull UpdateData<Integer, SerializedStubTree> updateData)
-      throws StorageException {
+    public void flush() throws StorageException {
+      final StubIndexImpl stubIndex = getStubIndex();
+      try {
+        stubIndex.flush();
+      }
+      finally {
+        super.flush();
+      }
+    }
 
+
+    @Override
+    protected void updateWithMap(int inputId,
+                                 @NotNull UpdateData<Integer, SerializedStubTree> updateData) throws StorageException {
       checkNameStorage();
       StubUpdatingData stubUpdatingData = (StubUpdatingData)updateData;
       final Map<StubIndexKey, Map<Object, StubIdList>> newStubIndicesValueMap = stubUpdatingData.getNewStubIndicesValueMap();

platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java

@@ -19,6 +19,7 @@
  */
 package com.intellij.util.indexing;
 
+import com.intellij.util.indexing.impl.IndexStorage;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java

@@ -76,13 +76,17 @@ import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.containers.JBIterable;
 import com.intellij.util.gist.GistManager;
 import com.intellij.util.gist.GistManagerImpl;
-import com.intellij.util.indexing.containers.TroveSetIntIterator;
+import com.intellij.util.indexing.impl.InvertedIndexValueIterator;
+import com.intellij.util.indexing.impl.MapReduceIndex;
 import com.intellij.util.io.DataOutputStream;
 import com.intellij.util.io.IOUtil;
 import com.intellij.util.io.storage.HeavyProcessLatch;
 import com.intellij.util.messages.MessageBus;
 import com.intellij.util.messages.MessageBusConnection;
-import gnu.trove.*;
+import gnu.trove.THashMap;
+import gnu.trove.THashSet;
+import gnu.trove.TIntArrayList;
+import gnu.trove.TIntHashSet;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;
@@ -350,7 +354,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
 
   private static <K, V> void initIndexStorage(@NotNull FileBasedIndexExtension<K, V> extension, int version, @NotNull File versionFile, IndexConfiguration state)
     throws IOException {
-    MapIndexStorage<K, V> storage = null;
+    VfsAwareMapIndexStorage<K, V> storage = null;
     final ID<K, V> name = extension.getName();
     boolean contentHashesEnumeratorOk = false;
 
@@ -360,7 +364,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
           ContentHashesSupport.initContentHashesEnumerator();
           contentHashesEnumeratorOk = true;
         }
-        storage = new MapIndexStorage<>(
+        storage = new VfsAwareMapIndexStorage<>(
           IndexInfrastructure.getStorageFile(name),
           extension.getKeyDescriptor(),
           extension.getValueExternalizer(),
@@ -444,17 +448,17 @@ public class FileBasedIndexImpl extends FileBasedIndex {
   private static <K, V> UpdatableIndex<K, V, FileContent> createIndex(@NotNull final FileBasedIndexExtension<K, V> extension,
                                                                @NotNull final MemoryIndexStorage<K, V> storage)
     throws StorageException, IOException {
-    final MapReduceIndex<K, V, FileContent> index;
+    final VfsAwareMapReduceIndex<K, V, FileContent> index;
     if (extension instanceof CustomImplementationFileBasedIndexExtension) {
       final UpdatableIndex<K, V, FileContent> custom =
         ((CustomImplementationFileBasedIndexExtension<K, V, FileContent>)extension).createIndexImplementation(extension, storage);
-      if (!(custom instanceof MapReduceIndex)) {
+      if (!(custom instanceof VfsAwareMapReduceIndex)) {
         return custom;
       }
-      index = (MapReduceIndex<K, V, FileContent>)custom;
+      index = (VfsAwareMapReduceIndex<K, V, FileContent>)custom;
     }
     else {
-      index = new MapReduceIndex<>(extension, storage);
+      index = new VfsAwareMapReduceIndex<>(extension, storage);
     }
 
     return index;
@@ -1046,75 +1050,15 @@ public class FileBasedIndexImpl extends FileBasedIndex {
                                                              @Nullable final Condition<V> valueChecker,
                                                              @Nullable final ProjectIndexableFilesFilter projectFilesFilter) {
     ThrowableConvertor<UpdatableIndex<K, V, FileContent>, TIntHashSet, StorageException> convertor =
-      index -> collectInputIdsContainingAllKeys(index, dataKeys, valueChecker,
+      index -> InvertedIndexUtil.collectInputIdsContainingAllKeys(index, dataKeys, (k) -> {
+                                                                    ProgressManager.checkCanceled();
+                                                                    return true;
+                                                                  }, valueChecker,
                                                 projectFilesFilter == null ? null : projectFilesFilter::containsFileId);
 
     return processExceptions(indexId, null, filter, convertor);
   }
 
-  @Nullable
-  private static <K, V, I> TIntHashSet collectInputIdsContainingAllKeys(@NotNull InvertedIndex<K, V, I> index,
-                                                                        @NotNull Collection<K> dataKeys,
-                                                                        @Nullable Condition<V> valueChecker,
-                                                                        @Nullable IntPredicate idChecker)
-    throws StorageException {
-    TIntHashSet mainIntersection = null;
-
-    for (K dataKey : dataKeys) {
-      ProgressManager.checkCanceled();
-      final TIntHashSet copy = new TIntHashSet();
-      final ValueContainer<V> container = index.getData(dataKey);
-
-      for (InvertedIndexValueIterator<V> valueIt = (InvertedIndexValueIterator<V>)container.getValueIterator(); valueIt.hasNext(); ) {
-        final V value = valueIt.next();
-        if (valueChecker != null && !valueChecker.value(value)) {
-          continue;
-        }
-
-        ValueContainer.IntIterator iterator = valueIt.getInputIdsIterator();
-
-        if (mainIntersection == null || iterator.size() < mainIntersection.size()) {
-          while (iterator.hasNext()) {
-            final int id = iterator.next();
-            if (mainIntersection == null && (idChecker == null || idChecker.contains(id)) ||
-                mainIntersection != null && mainIntersection.contains(id)
-              ) {
-              copy.add(id);
-            }
-          }
-        }
-        else {
-          mainIntersection.forEach(new TIntProcedure() {
-            final IntPredicate predicate = valueIt.getValueAssociationPredicate();
-
-            @Override
-            public boolean execute(int id) {
-              if (predicate.contains(id)) copy.add(id);
-              return true;
-            }
-          });
-        }
-      }
-
-      mainIntersection = copy;
-      if (mainIntersection.isEmpty()) {
-        return new TIntHashSet();
-      }
-    }
-
-    return mainIntersection;
-  }
-
-
-  @NotNull
-  public static <K, V, I> ValueContainer.IntIterator collectInputIdsContainingAllKeys(@NotNull InvertedIndex<K, V, I> index,
-                                                                                      @NotNull Collection<K> dataKeys)
-    throws StorageException {
-    TIntHashSet result = collectInputIdsContainingAllKeys(index, dataKeys, null, null);
-    if (result == null) return TroveSetIntIterator.EMPTY;
-    return new TroveSetIntIterator(result);
-  }
-
   private static boolean processVirtualFiles(@NotNull TIntHashSet ids,
                                              @NotNull final GlobalSearchScope filter,
                                              @NotNull final Processor<VirtualFile> processor) {

platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java

@@ -1,179 +0,0 @@
-/*
- * Copyright 2000-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.util.indexing;
-
-import com.intellij.openapi.application.ApplicationManager;
-import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.openapi.util.Comparing;
-import com.intellij.openapi.util.Ref;
-import com.intellij.util.SystemProperties;
-import gnu.trove.THashMap;
-import gnu.trove.TObjectObjectProcedure;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Value> {
-  public static boolean ourDiffUpdateEnabled = SystemProperties.getBooleanProperty("idea.disable.diff.index.update", true);
-
-  private Map<Key, Value> removedOrChangedKeys;
-  private Map<Key, Value> addedKeys;
-
-  public MapDiffUpdateData(ID<Key, Value> indexId) {
-    super(indexId);
-  }
-
-  public static <Key, Value> void iterateAddedKeyAndValues(final int inputId,
-                                                           final AddedKeyProcessor<Key, Value> consumer,
-                                                           Map<Key, Value> data) throws StorageException {
-    if (data instanceof THashMap) {
-      // such map often (from IdIndex) contain 100x (avg ~240) of entries, also THashMap have no Entry inside so we optimize for gc too
-      final Ref<StorageException> exceptionRef = new Ref<>();
-      final boolean b = ((THashMap<Key, Value>)data).forEachEntry(new TObjectObjectProcedure<Key, Value>() {
-        @Override
-        public boolean execute(Key key, Value value) {
-          try {
-            consumer.process(key, value, inputId);
-          }
-          catch (StorageException ex) {
-            exceptionRef.set(ex);
-            return false;
-          }
-          return true;
-        }
-      });
-      if (!b) throw exceptionRef.get();
-    }
-    else {
-      for (Map.Entry<Key, Value> entry : data.entrySet()) {
-        consumer.process(entry.getKey(), entry.getValue(), inputId);
-      }
-    }
-  }
-
-  public static <Key> void iterateRemovedKeys(Collection<Key> keyCollection, int inputId,
-                                                                   RemovedOrUpdatedKeyProcessor<Key> consumer) throws StorageException {
-    for (Key key : keyCollection) {
-      consumer.process(key, inputId);
-    }
-  }
-
-  @Override
-  public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer)
-    throws StorageException {
-    calcDiff();
-    iterateRemovedKeys(removedOrChangedKeys.keySet(), inputId, consumer);
-  }
-
-  private static final boolean DO_INFO_DUMP = ApplicationManager.getApplication().isInternal();
-
-  private void calcDiff() throws StorageException {
-    if (removedOrChangedKeys != null) return;
-
-    try {
-      Map<Key, Value> currentValue = getCurrentValue();
-      Map<Key, Value> newValue = getNewValue();
-
-      if (!currentValue.isEmpty()) {
-        if (newValue.isEmpty()) {
-          // removal from index
-          addedKeys = newValue;
-          removedOrChangedKeys = currentValue;
-          return;
-        }
-        for (Map.Entry<Key, Value> e : currentValue.entrySet()) {
-          Value newValueForKey = newValue.get(e.getKey());
-
-          if (!Comparing.equal(newValueForKey, e.getValue()) ||
-              newValueForKey == null && !newValue.containsKey(e.getKey())
-            ) {
-            if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap<>();
-            removedOrChangedKeys.put(e.getKey(), e.getValue());
-            if (newValue.containsKey(e.getKey())) {
-              if (addedKeys == null) addedKeys = new THashMap<>();
-              addedKeys.put(e.getKey(), newValueForKey);
-            }
-          }
-        }
-      }
-      else {
-        if (newValue.isEmpty()) {
-          // before and after map are empty
-          addedKeys = newValue;
-          removedOrChangedKeys = currentValue;
-          return;
-        }
-      }
-
-      if (!newValue.isEmpty()) {
-        if (currentValue.isEmpty()) {
-          // initial indexing
-          addedKeys = newValue;
-          removedOrChangedKeys = currentValue;
-          return;
-        }
-        for (Map.Entry<Key, Value> e : newValue.entrySet()) {
-          if (!currentValue.containsKey(e.getKey())) {
-            if (addedKeys == null) addedKeys = new THashMap<>();
-            addedKeys.put(e.getKey(), e.getValue());
-          }
-        }
-      }
-
-      if (removedOrChangedKeys == null) removedOrChangedKeys = Collections.emptyMap();
-      if (addedKeys == null) addedKeys = Collections.emptyMap();
-
-      int totalRequests = requests.incrementAndGet();
-      totalRemovals.addAndGet(currentValue.size());
-      totalAdditions.addAndGet(newValue.size());
-      incrementalAdditions.addAndGet(addedKeys.size());
-      incrementalRemovals.addAndGet(removedOrChangedKeys.size());
-
-      if ((totalRequests & 0xFFF) == 0 && DO_INFO_DUMP) {
-        Logger.getInstance(getClass()).info("Incremental index diff update:"+requests +
-                                            ", removals:" + totalRemovals + "->" + incrementalRemovals +
-                                            ", additions:" +totalAdditions + "->" +incrementalAdditions);
-      }
-      //if (removedOrChangedKeys.size() != currentValue.size() ||
-      //    addedKeys.size() != newValue.size()
-      //   ) {
-      //  int a = 1; // your breakpoint can be here
-      //}
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
-    }
-  }
-
-  private static final AtomicInteger requests = new AtomicInteger();
-  private static final AtomicInteger totalRemovals = new AtomicInteger();
-  private static final AtomicInteger totalAdditions = new AtomicInteger();
-  private static final AtomicInteger incrementalRemovals = new AtomicInteger();
-  private static final AtomicInteger incrementalAdditions = new AtomicInteger();
-
-  protected abstract Map<Key, Value> getNewValue();
-
-  protected abstract Map<Key, Value> getCurrentValue() throws IOException;
-
-  @Override
-  public void iterateAddedKeys(int inputId, AddedKeyProcessor<Key, Value> consumer) throws StorageException {
-    calcDiff();
-    iterateAddedKeyAndValues(inputId, consumer, addedKeys);
-  }
-}

platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java

@@ -1,994 +0,0 @@
-/*
- * Copyright 2000-2014 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.intellij.util.indexing;
-
-import com.intellij.openapi.application.Application;
-import com.intellij.openapi.application.ApplicationManager;
-import com.intellij.openapi.application.ModalityState;
-import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.openapi.editor.Document;
-import com.intellij.openapi.fileEditor.FileDocumentManager;
-import com.intellij.openapi.fileTypes.FileType;
-import com.intellij.openapi.progress.ProcessCanceledException;
-import com.intellij.openapi.progress.ProgressManager;
-import com.intellij.openapi.util.*;
-import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
-import com.intellij.openapi.util.io.ByteSequence;
-import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
-import com.intellij.psi.PsiDocumentManager;
-import com.intellij.psi.PsiFile;
-import com.intellij.psi.impl.cache.impl.id.IdIndex;
-import com.intellij.psi.search.GlobalSearchScope;
-import com.intellij.util.*;
-import com.intellij.util.io.*;
-import com.intellij.util.io.DataOutputStream;
-import gnu.trove.THashMap;
-import gnu.trove.TIntObjectHashMap;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.io.*;
-import java.nio.charset.Charset;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * @author Eugene Zhuravlev
- *         Date: Dec 10, 2007
- */
-public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Value, Input> {
-  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
-  private static final int NULL_MAPPING = 0;
-  @NotNull private final ID<Key, Value> myIndexId;
-  private final DataIndexer<Key, Value, Input> myIndexer;
-  @NotNull protected final IndexStorage<Key, Value> myStorage;
-  private final boolean myHasSnapshotMapping;
-
-  private final DataExternalizer<Value> myValueExternalizer;
-  private final DataExternalizer<Collection<Key>> mySnapshotIndexExternalizer;
-  private final boolean myIsPsiBackedIndex;
-  private final IndexExtension<Key, Value, Input> myExtension;
-  private final AtomicBoolean myInMemoryMode = new AtomicBoolean();
-  private final AtomicLong myModificationStamp = new AtomicLong();
-  private final TIntObjectHashMap<Collection<Key>> myInMemoryKeys = new TIntObjectHashMap<>();
-
-  private PersistentHashMap<Integer, ByteSequence> myContents;
-  private PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
-  @Nullable protected PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
-  private PersistentHashMap<Integer, String> myIndexingTrace;
-  private volatile boolean myDisposed;
-
-  private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
-
-  private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() {
-    @Override
-    public void run() {
-      try {
-        if (myStorage instanceof MemoryIndexStorage) {
-          Lock writeLock = getWriteLock();
-          if (writeLock.tryLock()) {
-            try {
-              ((MemoryIndexStorage<Key, Value>)myStorage).clearCaches();
-            } finally {
-              writeLock.unlock();
-            }
-          }
-        }
-        flush();
-      } catch (StorageException e) {
-        LOG.info(e);
-        requestRebuild(null);
-      }
-    }
-  });
-
-  public MapReduceIndex(IndexExtension<Key, Value, Input> extension,
-                        @NotNull IndexStorage<Key, Value> storage) throws IOException {
-    myIndexId = extension.getName();
-    myExtension = extension;
-    SharedIndicesData.registerIndex(myIndexId, extension);
-    myIndexer = extension.getIndexer();
-    myStorage = storage;
-    myHasSnapshotMapping = extension instanceof FileBasedIndexExtension &&
-                           ((FileBasedIndexExtension<Key, Value>)extension).hasSnapshotMapping() &&
-                           IdIndex.ourSnapshotMappingsEnabled;
-
-    mySnapshotIndexExternalizer = createInputsIndexExternalizer(extension, myIndexId, extension.getKeyDescriptor());
-    myValueExternalizer = extension.getValueExternalizer();
-    myIsPsiBackedIndex = extension instanceof PsiDependentIndex;
-
-    myContents = createContentsIndex(); // todo
-
-    if (!SharedIndicesData.ourFileSharedIndicesEnabled || SharedIndicesData.DO_CHECKS) {
-      if (myHasSnapshotMapping) {
-        myInputsSnapshotMapping = createInputSnapshotMapping();
-      }
-      else {
-        myInputsIndex = createInputsIndex();
-      }
-    }
-
-    if (DebugAssertions.EXTRA_SANITY_CHECKS && myHasSnapshotMapping) {
-      myIndexingTrace = createIndexingTrace();
-    }
-
-    if (storage instanceof MemoryIndexStorage) {
-      ((MemoryIndexStorage)storage).addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() {
-        @Override
-        public void bufferingStateChanged(boolean newState) {
-          myInMemoryMode.set(newState);
-        }
-
-        @Override
-        public void memoryStorageCleared() {
-          synchronized (myInMemoryKeys) {
-            myInMemoryKeys.clear();
-          }
-        }
-      });
-    }
-  }
-
-  private static <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
-                                                                           ID<K, ?> indexId,
-                                                                           KeyDescriptor<K> keyDescriptor) {
-    DataExternalizer<Collection<K>> externalizer;
-    if (extension instanceof CustomInputsIndexFileBasedIndexExtension) {
-      externalizer = ((CustomInputsIndexFileBasedIndexExtension<K>)extension).createExternalizer();
-    } else {
-      externalizer = new InputIndexDataExternalizer<>(keyDescriptor, indexId);
-    }
-    return externalizer;
-  }
-
-  @NotNull
-  private static <K> PersistentHashMap<Integer, Collection<K>> createIdToDataKeysIndex(@NotNull IndexExtension <K, ?, ?> extension,
-                                                                                      @NotNull MemoryIndexStorage<K, ?> storage)
-    throws IOException {
-    ID<K, ?> indexId = extension.getName();
-    KeyDescriptor<K> keyDescriptor = extension.getKeyDescriptor();
-    final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(indexId);
-
-    return new PersistentHashMap<>(
-      indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, createInputsIndexExternalizer(extension, indexId, keyDescriptor)
-    );
-  }
-
-  private PersistentHashMap<Integer, ByteSequence> createContentsIndex() throws IOException {
-    final File saved = myHasSnapshotMapping ? new File(IndexInfrastructure.getPersistentIndexRootDir(myIndexId), "values") : null;
-
-    if (saved != null) {
-      try {
-        return new PersistentHashMap<>(saved, EnumeratorIntegerDescriptor.INSTANCE, ByteSequenceDataExternalizer.INSTANCE);
-      } catch (IOException ex) {
-        IOUtil.deleteAllFilesStartingWith(saved);
-        throw ex;
-      }
-    } else {
-      return null;
-    }
-  }
-
-  @NotNull
-  public IndexStorage<Key, Value> getStorage() {
-    return myStorage;
-  }
-
-  @Override
-  public void clear() throws StorageException {
-    try {
-      getWriteLock().lock();
-      myStorage.clear();
-      if (myInputsIndex != null) {
-        cleanMapping(myInputsIndex);
-        myInputsIndex = createInputsIndex();
-      }
-      if (myInputsSnapshotMapping != null) {
-        cleanMapping(myInputsSnapshotMapping);
-        myInputsSnapshotMapping = createInputSnapshotMapping();
-      }
-      if (myIndexingTrace != null) {
-        cleanMapping(myIndexingTrace);
-        myIndexingTrace = createIndexingTrace();
-      }
-      if (myContents != null) {
-        cleanMapping(myContents);
-        myContents = createContentsIndex();
-      }
-    }
-    catch (StorageException e) {
-      LOG.error(e);
-    }
-    catch (IOException e) {
-      LOG.error(e);
-    }
-    finally {
-      getWriteLock().unlock();
-    }
-  }
-
-  private PersistentHashMap<Integer, Integer> createInputSnapshotMapping() throws IOException {
-    final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
-    try {
-      return new PersistentHashMap<Integer, Integer>(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE,
-                                                     EnumeratorIntegerDescriptor.INSTANCE, 4096) {
-        @Override
-        protected boolean wantNonnegativeIntegralValues() {
-          return true;
-        }
-      };
-    }
-    catch (IOException ex) {
-      IOUtil.deleteAllFilesStartingWith(fileIdToHashIdFile);
-      throw ex;
-    }
-  }
-
-  private PersistentHashMap<Integer, String> createIndexingTrace() throws IOException {
-    final File mapFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "indextrace");
-    try {
-      return new PersistentHashMap<>(mapFile, EnumeratorIntegerDescriptor.INSTANCE,
-                                     new DataExternalizer<String>() {
-                                       @Override
-                                       public void save(@NotNull DataOutput out, String value) throws IOException {
-                                         out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset()));
-                                       }
-
-                                       @Override
-                                       public String read(@NotNull DataInput in) throws IOException {
-                                         byte[] b = new byte[((InputStream)in).available()];
-                                         in.readFully(b);
-                                         return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset());
-                                       }
-                                     }, 4096);
-    }
-    catch (IOException ex) {
-      IOUtil.deleteAllFilesStartingWith(mapFile);
-      throw ex;
-    }
-  }
-
-  private static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) {
-    final File baseFile = index.getBaseFile();
-    try {
-      index.close();
-    }
-    catch (Throwable ignored) {
-    }
-
-    IOUtil.deleteAllFilesStartingWith(baseFile);
-  }
-
-  @Override
-  public void flush() throws StorageException{
-    try {
-      getReadLock().lock();
-      doForce(myInputsIndex);
-      doForce(myInputsSnapshotMapping);
-      doForce(myIndexingTrace);
-      doForce(myContents);
-      myStorage.flush();
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
-    }
-    catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof StorageException || cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      else {
-        throw e;
-      }
-    }
-    finally {
-      getReadLock().unlock();
-    }
-  }
-
-  private static void doForce(@Nullable PersistentHashMap<?, ?> inputsIndex) {
-    if (inputsIndex != null && inputsIndex.isDirty()) {
-      inputsIndex.force();
-    }
-  }
-
-  @Override
-  public void dispose() {
-    myLowMemoryFlusher.stop();
-    final Lock lock = getWriteLock();
-    try {
-      lock.lock();
-      try {
-        myStorage.close();
-      }
-      finally {
-        doClose(myInputsIndex);
-        doClose(myInputsSnapshotMapping);
-        doClose(myIndexingTrace);
-        doClose(myContents);
-      }
-    }
-    catch (StorageException e) {
-      LOG.error(e);
-    }
-    finally {
-      myDisposed = true;
-      lock.unlock();
-    }
-  }
-
-  @Override
-  public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
-    IndexingStamp.setFileIndexedStateCurrent(fileId, myIndexId);
-  }
-
-  @Override
-  public void resetIndexedStateForFile(int fileId) {
-    IndexingStamp.setFileIndexedStateOutdated(fileId, myIndexId);
-  }
-
-  @Override
-  public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
-    return IndexingStamp.isFileIndexedStateCurrent(fileId, myIndexId);
-  }
-
-  private static void doClose(@Nullable PersistentHashMap<?, ?> index) {
-    if (index != null) {
-      try {
-        index.close();
-      }
-      catch (IOException e) {
-        LOG.error(e);
-      }
-    }
-  }
-
-  @NotNull
-  @Override
-  public final Lock getReadLock() {
-    return myLock.readLock();
-  }
-
-  @NotNull
-  @Override
-  public final Lock getWriteLock() {
-    return myLock.writeLock();
-  }
-
-  @Override
-  public boolean processAllKeys(@NotNull Processor<Key> processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
-    final Lock lock = getReadLock();
-    try {
-      lock.lock();
-      return myStorage.processKeys(processor, scope, idFilter);
-    }
-    finally {
-      lock.unlock();
-    }
-  }
-
-  @Override
-  @NotNull
-  public ValueContainer<Value> getData(@NotNull final Key key) throws StorageException {
-    final Lock lock = getReadLock();
-    try {
-      lock.lock();
-      if (myDisposed) {
-        return new ValueContainerImpl<>();
-      }
-      ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
-      return myStorage.read(key);
-    }
-    finally {
-      ValueContainerImpl.ourDebugIndexInfo.set(null);
-      lock.unlock();
-    }
-  }
-
-  protected PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException {
-    return createIdToDataKeysIndex(myExtension, (MemoryIndexStorage<Key, ?>)myStorage);
-  }
-
-  private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);
-
-  @NotNull
-  @Override
-  public final Computable<Boolean> update(final int inputId, @Nullable Input content) {
-    final boolean weProcessPhysicalContent = content == null ||
-                                             (content instanceof UserDataHolder &&
-                                              FileBasedIndexImpl.ourPhysicalContentKey.get((UserDataHolder)content, Boolean.FALSE));
-
-    Map<Key, Value> data = null;
-    boolean havePersistentData = false;
-    Integer hashId = null;
-    boolean skippedReadingPersistentDataButMayHaveIt = false;
-
-    if (myContents != null && weProcessPhysicalContent && content != null) {
-      try {
-        FileContent fileContent = (FileContent)content;
-        hashId = getHashOfContent(fileContent);
-        if (doReadSavedPersistentData) {
-          if (!myContents.isBusyReading() || DebugAssertions.EXTRA_SANITY_CHECKS) { // avoid blocking read, we can calculate index value
-            ByteSequence bytes = readContents(hashId);
-
-            if (bytes != null) {
-              data = deserializeSavedPersistentData(bytes);
-              havePersistentData = true;
-              if (DebugAssertions.EXTRA_SANITY_CHECKS) {
-                Map<Key, Value> contentData = myIndexer.map(content);
-                boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data);
-                if (!sameValueForSavedIndexedResultAndCurrentOne) {
-                  DebugAssertions.error(
-                    "Unexpected difference in indexing of %s by index %s, file type %s, charset %s\ndiff %s\nprevious indexed info %s",
-                    fileContent.getFile(),
-                    myIndexId,
-                    fileContent.getFileType().getName(),
-                    ((FileContentImpl)fileContent).getCharset(),
-                    buildDiff(data, contentData),
-                    myIndexingTrace.get(hashId)
-                  );
-                }
-              }
-            }
-          } else {
-            skippedReadingPersistentDataButMayHaveIt = true;
-          }
-        } else {
-          havePersistentData = myContents.containsMapping(hashId);
-        }
-      } catch (IOException ex) {
-        // todo:
-        throw new RuntimeException(ex);
-      }
-    }
-
-    if (data == null) {
-      data = content != null ? myIndexer.map(content) : Collections.emptyMap();
-      if (DebugAssertions.DEBUG) {
-        checkValuesHaveProperEqualsAndHashCode(data);
-      }
-    }
-
-    if (hashId != null && !havePersistentData) {
-      boolean saved = savePersistentData(data, hashId, skippedReadingPersistentDataButMayHaveIt);
-      if (DebugAssertions.EXTRA_SANITY_CHECKS) {
-        if (saved) {
-
-          FileContent fileContent = (FileContent)content;
-          try {
-            myIndexingTrace.put(hashId, ((FileContentImpl)fileContent).getCharset() + "," + fileContent.getFileType().getName()+"," + fileContent.getFile().getPath() + "," +
-                                        ExceptionUtil.getThrowableText(new Throwable()));
-          } catch (IOException ex) {
-            LOG.error(ex);
-          }
-        }
-      }
-    }
-    ProgressManager.checkCanceled();
-
-    UpdateData<Key, Value> optimizedUpdateData = null;
-    final NotNullComputable<Collection<Key>> oldKeysGetter;
-    final int savedInputId;
-    if (myHasSnapshotMapping) {
-      try {
-        final NotNullComputable<Collection<Key>> keysForGivenInputId = () -> {
-          try {
-            Integer currentHashId = readInputHashId(inputId);
-            Collection<Key> currentKeys;
-            if (currentHashId != null) {
-              ByteSequence byteSequence = readContents(currentHashId);
-              currentKeys = byteSequence != null ? deserializeSavedPersistentData(byteSequence).keySet() : Collections.<Key>emptyList();
-            }
-            else {
-              currentKeys = Collections.emptyList();
-            }
-
-            return currentKeys;
-          }
-          catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        };
-
-        if (weProcessPhysicalContent) {
-          if (content instanceof FileContent) {
-            savedInputId = getHashOfContent((FileContent)content);
-          }
-          else {
-            savedInputId = NULL_MAPPING;
-          }
-          oldKeysGetter = keysForGivenInputId;
-
-          if (MapDiffUpdateData.ourDiffUpdateEnabled) {
-            final Map<Key, Value> newValue = data;
-            optimizedUpdateData = new MapDiffUpdateData<Key, Value>(myIndexId) {
-              @Override
-              protected Map<Key, Value> getNewValue() {
-                return newValue;
-              }
-
-              @Override
-              protected Map<Key, Value> getCurrentValue() throws IOException {
-                Integer currentHashId = readInputHashId(inputId);
-                Map<Key, Value> currentValue;
-                if (currentHashId != null) {
-                  ByteSequence byteSequence = readContents(currentHashId);
-                  currentValue = byteSequence != null ? deserializeSavedPersistentData(byteSequence) : Collections.<Key, Value>emptyMap();
-                }
-                else {
-                  currentValue = Collections.emptyMap();
-                }
-                return currentValue;
-              }
-
-              @Override
-              public void save(int inputId) throws IOException {
-                saveInputHashId(inputId, savedInputId);
-              }
-            };
-          }
-        } else {
-          oldKeysGetter = () -> {
-            try {
-              Collection<Key> oldKeys = readInputKeys(inputId);
-              if (oldKeys == null) {
-                return keysForGivenInputId.compute();
-              }
-              return oldKeys;
-            }
-            catch (IOException e) {
-              throw new RuntimeException(e);
-            }
-          };
-          savedInputId = NULL_MAPPING;
-        }
-      } catch (IOException ex) {
-        throw new RuntimeException(ex);
-      }
-    } else {
-      oldKeysGetter = () -> {
-        try {
-          Collection<Key> oldKeys = readInputKeys(inputId);
-          return oldKeys == null? Collections.<Key>emptyList() : oldKeys;
-        }
-        catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-      };
-      savedInputId = inputId;
-    }
-
-    // do not depend on content!
-    final UpdateData<Key, Value> updateData = optimizedUpdateData != null ? optimizedUpdateData : buildUpdateData(data, oldKeysGetter, savedInputId);
-    return () -> {
-
-      try {
-        updateWithMap(inputId, updateData);
-      }
-      catch (StorageException|ProcessCanceledException ex) {
-        LOG.info("Exception during updateWithMap:" + ex);
-        Application application = ApplicationManager.getApplication();
-        if (application.isUnitTestMode() || application.isHeadlessEnvironment()) {
-          // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask
-          application.invokeLater(() -> requestRebuild(ex), ModalityState.any());
-        } else {
-          requestRebuild(ex);
-        }
-        return Boolean.FALSE;
-      }
-
-      return Boolean.TRUE;
-    };
-  }
-
-  protected void requestRebuild(@Nullable Exception ex) {
-    if (ex == null) {
-      FileBasedIndex.getInstance().requestRebuild(myIndexId);
-    }
-    else {
-      FileBasedIndex.getInstance().requestRebuild(myIndexId, ex);
-    }
-  }
-
-  protected UpdateData<Key, Value> buildUpdateData(Map<Key, Value> data, NotNullComputable<Collection<Key>> oldKeysGetter, int savedInputId) {
-    return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter);
-  }
-
-  private ByteSequence readContents(Integer hashId) throws IOException {
-    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-      if (SharedIndicesData.DO_CHECKS) {
-        synchronized (myContents) {
-          ByteSequence contentBytes = SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE);
-          ByteSequence contentBytesFromContents = myContents.get(hashId);
-
-          if ((contentBytes == null && contentBytesFromContents != null) ||
-              !Comparing.equal(contentBytesFromContents, contentBytes)) {
-            SharedIndicesData.associateContentData(hashId, myIndexId, contentBytesFromContents, ByteSequenceDataExternalizer.INSTANCE);
-            if (contentBytes != null) {
-              LOG.error("Unexpected indexing diff with hashid " + myIndexId + "," + hashId);
-            }
-            contentBytes = contentBytesFromContents;
-          }
-          return contentBytes;
-        }
-      } else {
-        return SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE);
-      }
-    }
-
-    return myContents.get(hashId);
-  }
-
-  private void saveContents(int id, BufferExposingByteArrayOutputStream out) throws IOException {
-    ByteSequence byteSequence = new ByteSequence(out.getInternalBuffer(), 0, out.size());
-    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-      if (SharedIndicesData.DO_CHECKS) {
-        synchronized (myContents) {
-          myContents.put(id, byteSequence);
-          SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE);
-        }
-      } else {
-        SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE);
-      }
-    } else {
-      myContents.put(id, byteSequence);
-    }
-  }
-
-  private Integer readInputHashId(int inputId) throws IOException {
-    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-      Integer hashId = SharedIndicesData.recallFileData(inputId, myIndexId, EnumeratorIntegerDescriptor.INSTANCE);
-      if (hashId == null) hashId = 0;
-      if (myInputsSnapshotMapping == null) return hashId;
-
-      Integer hashIdFromInputSnapshotMapping = myInputsSnapshotMapping.get(inputId);
-      if ((hashId == 0 && hashIdFromInputSnapshotMapping != 0) ||
-          !Comparing.equal(hashIdFromInputSnapshotMapping, hashId)) {
-        SharedIndicesData.associateFileData(inputId, myIndexId, hashIdFromInputSnapshotMapping,
-                                            EnumeratorIntegerDescriptor.INSTANCE);
-        if (hashId != 0) {
-          LOG.error("Unexpected indexing diff with hashid " + myIndexId + ", file:" + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId)
-                    + "," + hashIdFromInputSnapshotMapping + "," + hashId);
-        }
-        hashId = hashIdFromInputSnapshotMapping;
-      }
-      return hashId;
-    }
-    return myInputsSnapshotMapping.get(inputId);
-  }
-
-  private void saveInputHashId(int inputId, int savedInputId) throws IOException {
-    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-      SharedIndicesData.associateFileData(inputId, myIndexId, savedInputId, EnumeratorIntegerDescriptor.INSTANCE);
-    }
-
-    if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, savedInputId);
-  }
-
-  private Collection<Key> readInputKeys(int inputId) throws IOException {
-    if (myInMemoryMode.get()) {
-      synchronized (myInMemoryKeys) {
-        Collection<Key> keys = myInMemoryKeys.get(inputId);
-        if (keys != null) {
-          return keys;
-        }
-      }
-    }
-    if (myHasSnapshotMapping) {
-      return null;
-    }
-
-    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-      Collection<Key> keys = SharedIndicesData.recallFileData(inputId, myIndexId, mySnapshotIndexExternalizer);
-      if (myInputsIndex != null) {
-        Collection<Key> keysFromInputsIndex = myInputsIndex.get(inputId);
-
-        if ((keys == null && keysFromInputsIndex != null) ||
-            !DebugAssertions.equals(keysFromInputsIndex, keys, myExtension.getKeyDescriptor())
-           ) {
-          SharedIndicesData.associateFileData(inputId, myIndexId, keysFromInputsIndex, mySnapshotIndexExternalizer);
-          if (keys != null) {
-            DebugAssertions.error(
-              "Unexpected indexing diff " + myIndexId + ", file:" + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId)
-              + "," + keysFromInputsIndex + "," + keys);
-          }
-          keys = keysFromInputsIndex;
-        }
-      }
-      return keys;
-    }
-    return myInputsIndex != null ? myInputsIndex.get(inputId) : null;
-  }
-
-  private void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
-    if (myInMemoryMode.get()) {
-      synchronized (myInMemoryKeys) {
-        myInMemoryKeys.put(inputId, newData.keySet());
-      }
-    } else {
-      if (myHasSnapshotMapping) {
-        saveInputHashId(inputId, savedInputId);
-      } else {
-        if (myInputsIndex != null) {
-          if (newData.size() > 0) {
-            myInputsIndex.put(inputId, newData.keySet());
-          }
-          else {
-            myInputsIndex.remove(inputId);
-          }
-        }
-
-        if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-          Set<Key> newKeys = newData.keySet();
-          if (newKeys.size() == 0) newKeys = null;
-          SharedIndicesData.associateFileData(inputId, myIndexId, newKeys, mySnapshotIndexExternalizer);
-        }
-      }
-    }
-  }
-
-  private void checkValuesHaveProperEqualsAndHashCode(Map<Key, Value> data) {
-    for(Map.Entry<Key, Value> e: data.entrySet()) {
-      final Value value = e.getValue();
-      if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) {
-        LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter");
-      }
-
-      if (myValueExternalizer != null) {
-        try {
-          final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
-          DataOutputStream outputStream = new DataOutputStream(out);
-          myValueExternalizer.save(outputStream, value);
-          outputStream.close();
-          final Value deserializedValue =
-            myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size())));
-
-          if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) {
-            LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter");
-          }
-        } catch (IOException ex) {
-          LOG.error(ex);
-        }
-      }
-    }
-  }
-
-  private StringBuilder buildDiff(Map<Key, Value> data, Map<Key, Value> contentData) {
-    StringBuilder moreInfo = new StringBuilder();
-    if (contentData.size() != data.size()) {
-      moreInfo.append("Indexer has different number of elements, previously ").append(data.size()).append(" after ")
-        .append(contentData.size()).append("\n");
-    } else {
-      moreInfo.append("total ").append(contentData.size()).append(" entries\n");
-    }
-
-    for(Map.Entry<Key, Value> keyValueEntry:contentData.entrySet()) {
-      if (!data.containsKey(keyValueEntry.getKey())) {
-        moreInfo.append("Previous data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n");
-      }
-      else {
-        Value value = data.get(keyValueEntry.getKey());
-        if (!Comparing.equal(keyValueEntry.getValue(), value)) {
-          moreInfo.append("Previous data has different value for key:").append(keyValueEntry.getKey()).append( ", new value ").append(keyValueEntry.getValue()).append( ", oldValue:").append(value).append("\n");
-        }
-      }
-    }
-
-    for(Map.Entry<Key, Value> keyValueEntry:data.entrySet()) {
-      if (!contentData.containsKey(keyValueEntry.getKey())) {
-        moreInfo.append("New data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n");
-      }
-      else {
-        Value value = contentData.get(keyValueEntry.getKey());
-        if (!Comparing.equal(keyValueEntry.getValue(), value)) {
-          moreInfo.append("New data has different value for key:").append(keyValueEntry.getKey()).append( " new value ").append(value).append( ", oldValue:").append(keyValueEntry.getValue()).append("\n");
-        }
-      }
-    }
-    return moreInfo;
-  }
-
-  private Map<Key, Value> deserializeSavedPersistentData(ByteSequence bytes) throws IOException {
-    DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes.getBytes(), bytes.getOffset(), bytes.getLength()));
-    int pairs = DataInputOutputUtil.readINT(stream);
-    if (pairs == 0) return Collections.emptyMap();
-    Map<Key, Value> result = new THashMap<>(pairs);
-    while (stream.available() > 0) {
-      Value value = myValueExternalizer.read(stream);
-      Collection<Key> keys = mySnapshotIndexExternalizer.read(stream);
-      for(Key k:keys) result.put(k, value);
-    }
-    return result;
-  }
-
-  private Integer getHashOfContent(FileContent content) throws IOException {
-    FileType fileType = content.getFileType();
-    if (myIsPsiBackedIndex && myHasSnapshotMapping && content instanceof FileContentImpl) {
-      // psi backed index should use existing psi to build index value (FileContentImpl.getPsiFileForPsiDependentIndex())
-      // so we should use different bytes to calculate hash(Id)
-      Integer previouslyCalculatedUncommittedHashId = content.getUserData(ourSavedUncommittedHashIdKey);
-
-      if (previouslyCalculatedUncommittedHashId == null) {
-        Document document = FileDocumentManager.getInstance().getCachedDocument(content.getFile());
-
-        if (document != null) {  // if document is not committed
-          PsiDocumentManager psiDocumentManager = PsiDocumentManager.getInstance(content.getProject());
-
-          if (psiDocumentManager.isUncommited(document)) {
-            PsiFile file = psiDocumentManager.getCachedPsiFile(document);
-            Charset charset = ((FileContentImpl)content).getCharset();
-
-            if (file != null) {
-              previouslyCalculatedUncommittedHashId = ContentHashesSupport
-                .calcContentHashIdWithFileType(file.getText().getBytes(charset), charset,
-                                               fileType);
-              content.putUserData(ourSavedUncommittedHashIdKey, previouslyCalculatedUncommittedHashId);
-            }
-          }
-        }
-      }
-      if (previouslyCalculatedUncommittedHashId != null) return previouslyCalculatedUncommittedHashId;
-    }
-
-    Integer previouslyCalculatedContentHashId = content.getUserData(ourSavedContentHashIdKey);
-    if (previouslyCalculatedContentHashId == null) {
-      byte[] hash = content instanceof FileContentImpl ? ((FileContentImpl)content).getHash():null;
-      if (hash == null) {
-        if (fileType.isBinary()) {
-          previouslyCalculatedContentHashId = ContentHashesSupport.calcContentHashId(content.getContent(), fileType);
-        } else {
-          Charset charset = content instanceof FileContentImpl ? ((FileContentImpl)content).getCharset() : null;
-          previouslyCalculatedContentHashId = ContentHashesSupport
-            .calcContentHashIdWithFileType(content.getContent(), charset, fileType);
-        }
-      } else {
-        previouslyCalculatedContentHashId =  ContentHashesSupport.enumerateHash(hash);
-      }
-      content.putUserData(ourSavedContentHashIdKey, previouslyCalculatedContentHashId);
-    }
-    return previouslyCalculatedContentHashId;
-  }
-
-  private static final ThreadLocalCachedByteArray ourSpareByteArray = new ThreadLocalCachedByteArray();
-
-  private boolean savePersistentData(Map<Key, Value> data, int id, boolean delayedReading) {
-    try {
-      if (delayedReading && myContents.containsMapping(id)) return false;
-      BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(ourSpareByteArray.getBuffer(4 * data.size()));
-      DataOutputStream stream = new DataOutputStream(out);
-      int size = data.size();
-      DataInputOutputUtil.writeINT(stream, size);
-
-      if (size > 0) {
-        THashMap<Value, List<Key>> values = new THashMap<>();
-        List<Key> keysForNullValue = null;
-        for (Map.Entry<Key, Value> e : data.entrySet()) {
-          Value value = e.getValue();
-
-          List<Key> keys = value != null ? values.get(value):keysForNullValue;
-          if (keys == null) {
-            if (value != null) values.put(value, keys = new SmartList<>());
-            else keys = keysForNullValue = new SmartList<>();
-          }
-          keys.add(e.getKey());
-        }
-
-        if (keysForNullValue != null) {
-          myValueExternalizer.save(stream, null);
-          mySnapshotIndexExternalizer.save(stream, keysForNullValue);
-        }
-
-        for(Value value:values.keySet()) {
-          myValueExternalizer.save(stream, value);
-          mySnapshotIndexExternalizer.save(stream, values.get(value));
-        }
-      }
-
-      saveContents(id, out);
-    } catch (IOException ex) {
-      throw new RuntimeException(ex);
-    }
-    return true;
-  }
-
-  private static final com.intellij.openapi.util.Key<Integer> ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id");
-  private static final com.intellij.openapi.util.Key<Integer> ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id");
-
-  public IndexExtension<Key, Value, Input> getExtension() {
-    return myExtension;
-  }
-
-  public long getModificationStamp() {
-    return myModificationStamp.get();
-  }
-
-  public class SimpleUpdateData extends UpdateData<Key, Value> {
-    private final int savedInputId;
-    private final @NotNull Map<Key, Value> newData;
-    protected final @NotNull NotNullComputable<Collection<Key>> oldKeysGetter;
-
-    public SimpleUpdateData(ID<Key,Value> indexId, int id, @NotNull Map<Key, Value> data, @NotNull NotNullComputable<Collection<Key>> getter) {
-      super(indexId);
-      savedInputId = id;
-      newData = data;
-      oldKeysGetter = getter;
-    }
-
-    public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer) throws StorageException {
-      MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer);
-    }
-
-    public void iterateAddedKeys(final int inputId, final AddedKeyProcessor<Key, Value> consumer) throws StorageException {
-      MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData);
-    }
-
-    @Override
-    public void save(int inputId) throws IOException {
-      saveInputKeys(inputId, savedInputId, newData);
-    }
-
-    public @NotNull Map<Key, Value> getNewData() {
-      return newData;
-    }
-  }
-
-  private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>
-    myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>() {
-    @Override
-    public void process(Key key, int inputId) throws StorageException {
-      myModificationStamp.incrementAndGet();
-      myStorage.removeAllValues(key, inputId);
-    }
-  };
-
-  private final MapDiffUpdateData.AddedKeyProcessor<Key, Value> myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor<Key, Value>() {
-    @Override
-    public void process(Key key, Value value, int inputId) throws StorageException {
-      myModificationStamp.incrementAndGet();
-      myStorage.addValue(key, inputId, value);
-    }
-  };
-
-  protected void updateWithMap(final int inputId,
-                               @NotNull UpdateData<Key, Value> updateData) throws StorageException {
-    getWriteLock().lock();
-    try {
-      try {
-        ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
-        updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation);
-        updateData.iterateAddedKeys(inputId, myAddedKeyProcessor);
-        updateData.save(inputId);
-      }
-      catch (ProcessCanceledException pce) {
-        throw pce; // extra care
-      }
-      catch (Throwable e) { // e.g. IOException, AssertionError
-        throw new StorageException(e);
-      }
-      finally {
-        ValueContainerImpl.ourDebugIndexInfo.set(null);
-      }
-    }
-    finally {
-      getWriteLock().unlock();
-    }
-  }
-}

platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java

@@ -18,8 +18,11 @@ package com.intellij.util.indexing;
 
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.Processor;
-import com.intellij.util.Processors;
 import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.indexing.impl.ChangeTrackingValueContainer;
+import com.intellij.util.indexing.impl.DebugAssertions;
+import com.intellij.util.indexing.impl.IndexStorage;
+import com.intellij.util.indexing.impl.UpdatableValueContainer;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
@@ -31,7 +34,7 @@ import java.util.*;
  * @author Eugene Zhuravlev
  *         Date: Dec 10, 2007
  */
-public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value> {
+public class MemoryIndexStorage<Key, Value> implements VfsAwareIndexStorage<Key, Value> {
   private final Map<Key, ChangeTrackingValueContainer<Value>> myMap = new HashMap<>();
   @NotNull
   private final IndexStorage<Key, Value> myBackendStorage;
@@ -91,7 +94,8 @@ public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value>
     }
   }
 
-  void clearCaches() {
+  @Override
+  public void clearCaches() {
     if (myMap.size() == 0) return;
 
     if (DebugAssertions.DEBUG) {
@@ -120,14 +124,6 @@ public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value>
     myBackendStorage.flush();
   }
 
-  @NotNull
-  @Override
-  public Collection<Key> getKeys() throws StorageException {
-    final Set<Key> keys = new HashSet<>();
-    processKeys(Processors.cancelableCollectProcessor(keys), null, null);
-    return keys;
-  }
-
   @Override
   public boolean processKeys(@NotNull final Processor<Key> processor, GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
     final Set<Key> stopList = new HashSet<>();
@@ -148,7 +144,7 @@ public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value>
       }
       stopList.add(key);
     }
-    return myBackendStorage.processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter);
+    return ((VfsAwareIndexStorage<Key, Value>) myBackendStorage).processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter);
   }
 
   @Override

platform/lang-impl/src/com/intellij/util/indexing/SharedIndicesData.java

@@ -39,7 +39,7 @@ public class SharedIndicesData {
   static final boolean ourFileSharedIndicesEnabled = SystemProperties.getBooleanProperty("idea.shared.input.index.enabled", false);
   static final boolean DO_CHECKS = ourFileSharedIndicesEnabled && SystemProperties.getBooleanProperty("idea.shared.input.index.checked", false);
 
-  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
+  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.MapReduceIndex");
 
   static {
     if (ourFileSharedIndicesEnabled) {

platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
+import com.intellij.util.indexing.impl.AbstractForwardIndex;
+import com.intellij.util.indexing.impl.CollectionInputKeyIterator;
+import com.intellij.util.indexing.impl.DebugAssertions;
+import com.intellij.util.indexing.impl.MapBasedForwardIndex;
+import com.intellij.util.io.DataExternalizer;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+class SharedMapBasedForwardIndex<Key, Value> extends AbstractForwardIndex<Key,Value> {
+  private final DataExternalizer<Collection<Key>> mySnapshotIndexExternalizer;
+  private MapBasedForwardIndex<Key, Value> myUnderlying;
+
+  public SharedMapBasedForwardIndex(MapBasedForwardIndex<Key, Value> underlying) {
+    super(underlying.getIndexExtension());
+    myUnderlying = underlying;
+    mySnapshotIndexExternalizer = VfsAwareMapReduceIndex.createInputsIndexExternalizer(underlying.getIndexExtension());
+  }
+
+  @NotNull
+  @Override
+  public InputKeyIterator<Key, Value> getInputKeys(int inputId) throws IOException {
+    Collection<Key> keys;
+    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+      keys = SharedIndicesData.recallFileData(inputId, myIndexId, mySnapshotIndexExternalizer);
+      Collection<Key> keysFromInputsIndex = myUnderlying.getInputsIndex().get(inputId);
+
+      if ((keys == null && keysFromInputsIndex != null) ||
+          !DebugAssertions.equals(keysFromInputsIndex, keys, myKeyDescriptor)
+        ) {
+        SharedIndicesData.associateFileData(inputId, myIndexId, keysFromInputsIndex, mySnapshotIndexExternalizer);
+        if (keys != null) {
+          DebugAssertions.error(
+            "Unexpected indexing diff " + myIndexId + ", file:" + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId)
+            + "," + keysFromInputsIndex + "," + keys);
+        }
+        keys = keysFromInputsIndex;
+      }
+      return new CollectionInputKeyIterator<>(keys);
+    }
+    return new CollectionInputKeyIterator<>(myUnderlying.getInputsIndex().get(inputId));
+  }
+
+  @Override
+  public void putInputData(int inputId, @NotNull Map<Key, Value> data)
+    throws IOException {
+    Collection<Key> keySeq = data.keySet();
+    myUnderlying.putData(inputId, keySeq);
+    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+      if (keySeq.size() == 0) keySeq = null;
+      SharedIndicesData.associateFileData(inputId, myIndexId, keySeq, mySnapshotIndexExternalizer);
+    }
+  }
+
+  @Override
+  public void flush() {
+    myUnderlying.flush();
+  }
+
+  @Override
+  public void clear() throws IOException {
+    myUnderlying.clear();
+  }
+
+  @Override
+  public void close() throws IOException {
+    myUnderlying.close();
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java

@@ -0,0 +1,489 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.editor.Document;
+import com.intellij.openapi.fileEditor.FileDocumentManager;
+import com.intellij.openapi.fileTypes.FileType;
+import com.intellij.openapi.util.Comparing;
+import com.intellij.openapi.util.ThreadLocalCachedByteArray;
+import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
+import com.intellij.openapi.util.io.ByteSequence;
+import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
+import com.intellij.psi.PsiDocumentManager;
+import com.intellij.psi.PsiFile;
+import com.intellij.util.CompressionUtil;
+import com.intellij.util.ExceptionUtil;
+import com.intellij.util.SmartList;
+import com.intellij.util.SystemProperties;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.indexing.impl.DebugAssertions;
+import com.intellij.util.indexing.impl.MapReduceIndex;
+import com.intellij.util.io.*;
+import com.intellij.util.io.DataOutputStream;
+import gnu.trove.THashMap;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class SnapshotInputMappings<Key, Value, Input> {
+  private static final Logger LOG = Logger.getInstance(SnapshotInputMappings.class);
+  private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);
+
+  private final ID<Key, Value> myIndexId;
+  private final DataExternalizer<Value> myValueExternalizer;
+  private final IndexExtension<Key, Value, Input> myIndexExtension;
+  private final DataIndexer<Key, Value, Input> myIndexer;
+  private volatile PersistentHashMap<Integer, ByteSequence> myContents;
+  private volatile PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
+  private volatile PersistentHashMap<Integer, String> myIndexingTrace;
+
+  private final DataExternalizer<Collection<Key>> mySnapshotIndexExternalizer;
+  private boolean myIsPsiBackedIndex;
+
+  public SnapshotInputMappings(IndexExtension<Key, Value, Input> indexExtension) throws IOException {
+    myIndexId = indexExtension.getName();
+    myIsPsiBackedIndex = indexExtension instanceof PsiDependentIndex;
+    mySnapshotIndexExternalizer = VfsAwareMapReduceIndex.createInputsIndexExternalizer(indexExtension);
+    myValueExternalizer = indexExtension.getValueExternalizer();
+    myIndexer = indexExtension.getIndexer();
+    myIndexExtension = indexExtension;
+    createMaps();
+  }
+
+  @NotNull
+  public Map<Key, Value> readInputKeys(int inputId) throws IOException {
+    Integer currentHashId = readInputHashId(inputId);
+    if (currentHashId != null) {
+      ByteSequence byteSequence = readContents(currentHashId);
+      if (byteSequence != null) {
+        return deserializeSavedPersistentData(byteSequence);
+      }
+    }
+    return Collections.emptyMap();
+  }
+
+  static class Snapshot<Key, Value> {
+    private final Map<Key, Value> myData;
+    private final int hashId;
+
+    private Snapshot(Map<Key, Value> data, int id) {
+      myData = data;
+      hashId = id;
+    }
+
+    public Map<Key, Value> getData() {
+      return myData;
+    }
+
+    public int getHashId() {
+      return hashId;
+    }
+  }
+
+  @NotNull
+  Snapshot<Key, Value> readPersistentDataOrMap(@NotNull Input content) {
+    Map<Key, Value> data = null;
+    boolean havePersistentData = false;
+    int hashId;
+    boolean skippedReadingPersistentDataButMayHaveIt = false;
+
+    try {
+      FileContent fileContent = (FileContent)content;
+      hashId = getHashOfContent(fileContent);
+      if (doReadSavedPersistentData) {
+        if (!myContents.isBusyReading() || DebugAssertions.EXTRA_SANITY_CHECKS) { // avoid blocking read, we can calculate index value
+          ByteSequence bytes = readContents(hashId);
+
+          if (bytes != null) {
+            data = deserializeSavedPersistentData(bytes);
+            havePersistentData = true;
+            if (DebugAssertions.EXTRA_SANITY_CHECKS) {
+              Map<Key, Value> contentData = myIndexer.map(content);
+              boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data);
+              if (!sameValueForSavedIndexedResultAndCurrentOne) {
+                DebugAssertions.error(
+                  "Unexpected difference in indexing of %s by index %s, file type %s, charset %s\ndiff %s\nprevious indexed info %s",
+                  fileContent.getFile(),
+                  myIndexId,
+                  fileContent.getFileType().getName(),
+                  ((FileContentImpl)fileContent).getCharset(),
+                  buildDiff(data, contentData),
+                  myIndexingTrace.get(hashId)
+                );
+              }
+            }
+          }
+        }
+        else {
+          skippedReadingPersistentDataButMayHaveIt = true;
+        }
+      }
+      else {
+        havePersistentData = myContents.containsMapping(hashId);
+      }
+    }
+    catch (IOException ex) {
+      // todo:
+      throw new RuntimeException(ex);
+    }
+
+    if (data == null) {
+      data = myIndexer.map(content);
+      if (DebugAssertions.DEBUG) {
+        MapReduceIndex.checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer);
+      }
+    }
+
+    if (!havePersistentData) {
+      boolean saved = savePersistentData(data, hashId, skippedReadingPersistentDataButMayHaveIt);
+      if (DebugAssertions.EXTRA_SANITY_CHECKS) {
+        if (saved) {
+
+          FileContent fileContent = (FileContent)content;
+          try {
+            myIndexingTrace.put(hashId, ((FileContentImpl)fileContent).getCharset() +
+                                        "," +
+                                        fileContent.getFileType().getName() +
+                                        "," +
+                                        fileContent.getFile().getPath() +
+                                        "," +
+                                        ExceptionUtil.getThrowableText(new Throwable()));
+          }
+          catch (IOException ex) {
+            LOG.error(ex);
+          }
+        }
+      }
+    }
+
+    return new Snapshot<>(data, hashId);
+  }
+
+  public void putInputHash(int inputId, int hashId)
+    throws IOException {
+    try {
+      if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+        SharedIndicesData.associateFileData(inputId, myIndexId, hashId, EnumeratorIntegerDescriptor.INSTANCE);
+      }
+      if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, hashId);
+    }
+    catch (IOException ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
+  public void flush() {
+    if (myContents != null) myContents.force();
+    if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.force();
+    if (myIndexingTrace != null) myIndexingTrace.force();
+  }
+
+  public void clear() throws IOException {
+    List<File> baseDirs = ContainerUtil.list(myContents, myIndexingTrace, myInputsSnapshotMapping)
+      .stream()
+      .filter(Objects::nonNull)
+      .map(PersistentHashMap::getBaseFile)
+      .collect(Collectors.toList());
+    try {
+      close();
+    }
+    catch (Exception e) {
+      LOG.error(e);
+    }
+    baseDirs.forEach(PersistentHashMap::deleteFilesStartingWith);
+    createMaps();
+  }
+
+  public void close() throws IOException {
+    if (myContents != null) myContents.close();
+    if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.close();
+    if (myIndexingTrace != null) myIndexingTrace.close();
+  }
+
+  private void createMaps() throws IOException {
+    myContents = createContentsIndex();
+    myIndexingTrace = DebugAssertions.EXTRA_SANITY_CHECKS ? createIndexingTrace() : null;
+    myInputsSnapshotMapping =
+      !SharedIndicesData.ourFileSharedIndicesEnabled || SharedIndicesData.DO_CHECKS ? createInputSnapshotMapping() : null;
+  }
+
+  private PersistentHashMap<Integer, ByteSequence> createContentsIndex() throws IOException {
+    final File saved = new File(IndexInfrastructure.getPersistentIndexRootDir(myIndexId), "values");
+    try {
+      return new PersistentHashMap<>(saved, EnumeratorIntegerDescriptor.INSTANCE, ByteSequenceDataExternalizer.INSTANCE);
+    }
+    catch (IOException ex) {
+      IOUtil.deleteAllFilesStartingWith(saved);
+      throw ex;
+    }
+  }
+
+  private PersistentHashMap<Integer, Integer> createInputSnapshotMapping() throws IOException {
+    final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
+    try {
+      return new PersistentHashMap<Integer, Integer>(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE,
+                                                     EnumeratorIntegerDescriptor.INSTANCE, 4096) {
+        @Override
+        protected boolean wantNonnegativeIntegralValues() {
+          return true;
+        }
+      };
+    }
+    catch (IOException ex) {
+      IOUtil.deleteAllFilesStartingWith(fileIdToHashIdFile);
+      throw ex;
+    }
+  }
+
+  private PersistentHashMap<Integer, String> createIndexingTrace() throws IOException {
+    final File mapFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "indextrace");
+    try {
+      return new PersistentHashMap<>(mapFile, EnumeratorIntegerDescriptor.INSTANCE,
+                                     new DataExternalizer<String>() {
+                                       @Override
+                                       public void save(@NotNull DataOutput out, String value) throws IOException {
+                                         out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset()));
+                                       }
+
+                                       @Override
+                                       public String read(@NotNull DataInput in) throws IOException {
+                                         byte[] b = new byte[((InputStream)in).available()];
+                                         in.readFully(b);
+                                         return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset());
+                                       }
+                                     }, 4096);
+    }
+    catch (IOException ex) {
+      IOUtil.deleteAllFilesStartingWith(mapFile);
+      throw ex;
+    }
+  }
+
+  private Integer readInputHashId(int inputId) throws IOException {
+    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+      Integer hashId = SharedIndicesData.recallFileData(inputId, myIndexId, EnumeratorIntegerDescriptor.INSTANCE);
+      if (hashId == null) hashId = 0;
+      if (myInputsSnapshotMapping == null) return hashId;
+
+      Integer hashIdFromInputSnapshotMapping = myInputsSnapshotMapping.get(inputId);
+      if ((hashId == 0 && hashIdFromInputSnapshotMapping != 0) ||
+          !Comparing.equal(hashIdFromInputSnapshotMapping, hashId)) {
+        SharedIndicesData.associateFileData(inputId, myIndexId, hashIdFromInputSnapshotMapping,
+                                            EnumeratorIntegerDescriptor.INSTANCE);
+        if (hashId != 0) {
+          LOG.error("Unexpected indexing diff with hashid " +
+                    myIndexId +
+                    ", file:" +
+                    IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId)
+                    +
+                    "," +
+                    hashIdFromInputSnapshotMapping +
+                    "," +
+                    hashId);
+        }
+        hashId = hashIdFromInputSnapshotMapping;
+      }
+      return hashId;
+    }
+    return myInputsSnapshotMapping.get(inputId);
+  }
+
+  private ByteSequence readContents(Integer hashId) throws IOException {
+    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+      if (SharedIndicesData.DO_CHECKS) {
+        synchronized (myContents) {
+          ByteSequence contentBytes = SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE);
+          ByteSequence contentBytesFromContents = myContents.get(hashId);
+
+          if ((contentBytes == null && contentBytesFromContents != null) ||
+              !Comparing.equal(contentBytesFromContents, contentBytes)) {
+            SharedIndicesData.associateContentData(hashId, myIndexId, contentBytesFromContents, ByteSequenceDataExternalizer.INSTANCE);
+            if (contentBytes != null) {
+              LOG.error("Unexpected indexing diff with hashid " + myIndexId + "," + hashId);
+            }
+            contentBytes = contentBytesFromContents;
+          }
+          return contentBytes;
+        }
+      } else {
+        return SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE);
+      }
+    }
+
+    return myContents.get(hashId);
+  }
+
+  private Map<Key, Value> deserializeSavedPersistentData(ByteSequence bytes) throws IOException {
+    DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes.getBytes(), bytes.getOffset(), bytes.getLength()));
+    int pairs = DataInputOutputUtil.readINT(stream);
+    if (pairs == 0) return Collections.emptyMap();
+    Map<Key, Value> result = new THashMap<>(pairs);
+    while (stream.available() > 0) {
+      Value value = myIndexExtension.getValueExternalizer().read(stream);
+      Collection<Key> keys = mySnapshotIndexExternalizer.read(stream);
+      for(Key k:keys) result.put(k, value);
+    }
+    return result;
+  }
+
+  private Integer getHashOfContent(FileContent content) throws IOException {
+    FileType fileType = content.getFileType();
+    if (myIsPsiBackedIndex && content instanceof FileContentImpl) {
+      // psi backed index should use existing psi to build index value (FileContentImpl.getPsiFileForPsiDependentIndex())
+      // so we should use different bytes to calculate hash(Id)
+      Integer previouslyCalculatedUncommittedHashId = content.getUserData(ourSavedUncommittedHashIdKey);
+
+      if (previouslyCalculatedUncommittedHashId == null) {
+        Document document = FileDocumentManager.getInstance().getCachedDocument(content.getFile());
+
+        if (document != null) {  // if document is not committed
+          PsiDocumentManager psiDocumentManager = PsiDocumentManager.getInstance(content.getProject());
+
+          if (psiDocumentManager.isUncommited(document)) {
+            PsiFile file = psiDocumentManager.getCachedPsiFile(document);
+            Charset charset = ((FileContentImpl)content).getCharset();
+
+            if (file != null) {
+              previouslyCalculatedUncommittedHashId = ContentHashesSupport
+                .calcContentHashIdWithFileType(file.getText().getBytes(charset), charset,
+                                               fileType);
+              content.putUserData(ourSavedUncommittedHashIdKey, previouslyCalculatedUncommittedHashId);
+            }
+          }
+        }
+      }
+      if (previouslyCalculatedUncommittedHashId != null) return previouslyCalculatedUncommittedHashId;
+    }
+
+    Integer previouslyCalculatedContentHashId = content.getUserData(ourSavedContentHashIdKey);
+    if (previouslyCalculatedContentHashId == null) {
+      byte[] hash = content instanceof FileContentImpl ? ((FileContentImpl)content).getHash():null;
+      if (hash == null) {
+        if (fileType.isBinary()) {
+          previouslyCalculatedContentHashId = ContentHashesSupport.calcContentHashId(content.getContent(), fileType);
+        } else {
+          Charset charset = content instanceof FileContentImpl ? ((FileContentImpl)content).getCharset() : null;
+          previouslyCalculatedContentHashId = ContentHashesSupport
+            .calcContentHashIdWithFileType(content.getContent(), charset, fileType);
+        }
+      } else {
+        previouslyCalculatedContentHashId =  ContentHashesSupport.enumerateHash(hash);
+      }
+      content.putUserData(ourSavedContentHashIdKey, previouslyCalculatedContentHashId);
+    }
+    return previouslyCalculatedContentHashId;
+  }
+  private static final com.intellij.openapi.util.Key<Integer> ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id");
+  private static final com.intellij.openapi.util.Key<Integer> ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id");
+
+
+  private StringBuilder buildDiff(Map<Key, Value> data, Map<Key, Value> contentData) {
+    StringBuilder moreInfo = new StringBuilder();
+    if (contentData.size() != data.size()) {
+      moreInfo.append("Indexer has different number of elements, previously ").append(data.size()).append(" after ")
+        .append(contentData.size()).append("\n");
+    } else {
+      moreInfo.append("total ").append(contentData.size()).append(" entries\n");
+    }
+
+    for(Map.Entry<Key, Value> keyValueEntry:contentData.entrySet()) {
+      if (!data.containsKey(keyValueEntry.getKey())) {
+        moreInfo.append("Previous data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n");
+      }
+      else {
+        Value value = data.get(keyValueEntry.getKey());
+        if (!Comparing.equal(keyValueEntry.getValue(), value)) {
+          moreInfo.append("Previous data has different value for key:").append(keyValueEntry.getKey()).append( ", new value ").append(keyValueEntry.getValue()).append( ", oldValue:").append(value).append("\n");
+        }
+      }
+    }
+
+    for(Map.Entry<Key, Value> keyValueEntry:data.entrySet()) {
+      if (!contentData.containsKey(keyValueEntry.getKey())) {
+        moreInfo.append("New data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n");
+      }
+      else {
+        Value value = contentData.get(keyValueEntry.getKey());
+        if (!Comparing.equal(keyValueEntry.getValue(), value)) {
+          moreInfo.append("New data has different value for key:").append(keyValueEntry.getKey()).append( " new value ").append(value).append( ", oldValue:").append(keyValueEntry.getValue()).append("\n");
+        }
+      }
+    }
+    return moreInfo;
+  }
+
+  private static final ThreadLocalCachedByteArray ourSpareByteArray = new ThreadLocalCachedByteArray();
+  private boolean savePersistentData(Map<Key, Value> data, int id, boolean delayedReading) {
+    try {
+      if (delayedReading && myContents.containsMapping(id)) return false;
+      BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(ourSpareByteArray.getBuffer(4 * data.size()));
+      DataOutputStream stream = new DataOutputStream(out);
+      int size = data.size();
+      DataInputOutputUtil.writeINT(stream, size);
+
+      if (size > 0) {
+        THashMap<Value, List<Key>> values = new THashMap<>();
+        List<Key> keysForNullValue = null;
+        for (Map.Entry<Key, Value> e : data.entrySet()) {
+          Value value = e.getValue();
+
+          List<Key> keys = value != null ? values.get(value):keysForNullValue;
+          if (keys == null) {
+            if (value != null) values.put(value, keys = new SmartList<>());
+            else keys = keysForNullValue = new SmartList<>();
+          }
+          keys.add(e.getKey());
+        }
+
+        if (keysForNullValue != null) {
+          myValueExternalizer.save(stream, null);
+          mySnapshotIndexExternalizer.save(stream, keysForNullValue);
+        }
+
+        for(Value value:values.keySet()) {
+          myValueExternalizer.save(stream, value);
+          mySnapshotIndexExternalizer.save(stream, values.get(value));
+        }
+      }
+
+      saveContents(id, out);
+    } catch (IOException ex) {
+      throw new RuntimeException(ex);
+    }
+    return true;
+  }
+
+  private void saveContents(int id, BufferExposingByteArrayOutputStream out) throws IOException {
+    ByteSequence byteSequence = new ByteSequence(out.getInternalBuffer(), 0, out.size());
+    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+      if (SharedIndicesData.DO_CHECKS) {
+        synchronized (myContents) {
+          myContents.put(id, byteSequence);
+          SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE);
+        }
+      } else {
+        SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE);
+      }
+    } else {
+      myContents.put(id, byteSequence);
+    }
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java

@@ -1,46 +0,0 @@
-/*
- * Copyright 2000-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.util.indexing;
-
-import java.io.IOException;
-
-public abstract class UpdateData<Key, Value> {
-  private final ID<Key, Value> myIndexId;
-
-  protected UpdateData(ID<Key, Value> indexId) {
-    myIndexId = indexId;
-  }
-
-  public abstract void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer)
-    throws StorageException;
-
-  public abstract void iterateAddedKeys(final int inputId, final AddedKeyProcessor<Key, Value> consumer) throws StorageException;
-
-  public abstract void save(int inputId) throws IOException;
-
-  public interface AddedKeyProcessor<Key, Value> {
-    void process(Key key, Value value, int inputId) throws StorageException;
-  }
-
-  public interface RemovedOrUpdatedKeyProcessor<Key> {
-    void process(Key key, int inputId) throws StorageException;
-  }
-
-  @Override
-  public String toString() {
-    return myIndexId + "," + getClass().getName();
-  }
-}

platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java

@@ -15,9 +15,12 @@
  */
 package com.intellij.util.indexing;
 
-/**
- * Created by Maxim.Mossienko on 11/22/2016.
- */
-public interface IntPredicate {
-  boolean contains(int id);
+import com.intellij.psi.search.GlobalSearchScope;
+import com.intellij.util.Processor;
+import com.intellij.util.indexing.impl.IndexStorage;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+public interface VfsAwareIndexStorage<Key, Value> extends IndexStorage<Key, Value> {
+  boolean processKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
 }

platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java

@@ -25,12 +25,11 @@ import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.search.ProjectAndLibrariesScope;
 import com.intellij.psi.search.ProjectScopeImpl;
 import com.intellij.util.Processor;
-import com.intellij.util.Processors;
 import com.intellij.util.SystemProperties;
 import com.intellij.util.ThrowableRunnable;
 import com.intellij.util.containers.ConcurrentIntObjectMap;
 import com.intellij.util.containers.ContainerUtil;
-import com.intellij.util.containers.SLRUCache;
+import com.intellij.util.indexing.impl.MapIndexStorage;
 import com.intellij.util.io.*;
 import com.intellij.util.io.DataOutputStream;
 import gnu.trove.TIntHashSet;
@@ -39,124 +38,49 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.*;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * @author Eugene Zhuravlev
  *         Date: Dec 20, 2007
  */
-public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
-  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
+public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<Key, Value> implements VfsAwareIndexStorage<Key, Value> {
+  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.MapIndexStorage");
   private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true);
   private final boolean myBuildKeyHashToVirtualFileMapping;
-  private PersistentMap<Key, UpdatableValueContainer<Value>> myMap;
   private AppendableStorageBackedByResizableMappedFile myKeyHashToVirtualFileMapping;
-  private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
   private volatile int myLastScannedId;
-  private final File myBaseStorageFile;
-  private final KeyDescriptor<Key> myKeyDescriptor;
-  private final int myCacheSize;
 
-  private final Lock l = new ReentrantLock();
-  private final DataExternalizer<Value> myDataExternalizer;
-  private final boolean myKeyIsUniqueForIndexedFile;
   private static final ConcurrentIntObjectMap<Boolean> ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap();
 
-  public MapIndexStorage(@NotNull File storageFile,
-                         @NotNull KeyDescriptor<Key> keyDescriptor,
-                         @NotNull DataExternalizer<Value> valueExternalizer,
-                         final int cacheSize
+  public VfsAwareMapIndexStorage(@NotNull File storageFile,
+                                 @NotNull KeyDescriptor<Key> keyDescriptor,
+                                 @NotNull DataExternalizer<Value> valueExternalizer,
+                                 final int cacheSize
   ) throws IOException {
     this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false);
   }
 
-  public MapIndexStorage(@NotNull File storageFile,
-                         @NotNull KeyDescriptor<Key> keyDescriptor,
-                         @NotNull DataExternalizer<Value> valueExternalizer,
-                         final int cacheSize,
-                         boolean keyIsUniqueForIndexedFile,
-                         boolean buildKeyHashToVirtualFileMapping) throws IOException {
-    myBaseStorageFile = storageFile;
-    myKeyDescriptor = keyDescriptor;
-    myCacheSize = cacheSize;
-    myDataExternalizer = valueExternalizer;
-    myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
+  public VfsAwareMapIndexStorage(@NotNull File storageFile,
+                                 @NotNull KeyDescriptor<Key> keyDescriptor,
+                                 @NotNull DataExternalizer<Value> valueExternalizer,
+                                 final int cacheSize,
+                                 boolean keyIsUniqueForIndexedFile,
+                                 boolean buildKeyHashToVirtualFileMapping) throws IOException {
+    super(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, false);
     myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
     initMapAndCache();
   }
 
-  private static final PersistentHashMapValueStorage.ExceptionalIOCancellationCallback ourProgressManagerCheckCancelledIOCanceller =
-    new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() {
-      @Override
-      public void checkCancellation() {
-        ProgressManager.checkCanceled();
-      }
-  };
-  private void initMapAndCache() throws IOException {
-    final ValueContainerMap<Key, Value> map;
-    PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(ourProgressManagerCheckCancelledIOCanceller);
-    PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
-    try {
-      map = new ValueContainerMap<>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
-    } finally {
-      PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
-      PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
-    }
-    myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
-      @Override
-      @NotNull
-      public ChangeTrackingValueContainer<Value> createValue(final Key key) {
-        return new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer<Value>() {
-          @NotNull
-          @Override
-          public Object getLock() {
-            return map.getDataAccessLock();
-          }
-
-          @Nullable
-          @Override
-          public ValueContainer<Value> compute() {
-            ValueContainer<Value> value;
-            try {
-              value = map.get(key);
-              if (value == null) {
-                value = new ValueContainerImpl<>();
-              }
-            }
-            catch (IOException e) {
-              throw new RuntimeException(e);
-            }
-            return value;
-          }
-        });
-      }
-
-      @Override
-      protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
-        if (valueContainer.isDirty()) {
-          try {
-            map.put(key, valueContainer);
-          }
-          catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        }
-      }
-    };
-
-    myMap = map;
-
+  @Override
+  protected void initMapAndCache() throws IOException {
+    super.initMapAndCache();
     myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ?
                                     new AppendableStorageBackedByResizableMappedFile(getProjectFile(), 4096, null, PagedFileStorage.MB, true) : null;
   }
 
-  @NotNull
-  private File getStorageFile() {
-    return new File(myBaseStorageFile.getPath() + ".storage");
+  @Override
+  protected void checkCanceled() {
+    ProgressManager.checkCanceled();
   }
 
   @NotNull
@@ -172,14 +96,12 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
       myKeyHashToVirtualFileMapping.getPagedFileStorage().unlock();
     }
   }
+
   @Override
   public void flush() {
     l.lock();
     try {
-      if (!myMap.isClosed()) {
-        myCache.clear();
-        if (myMap.isDirty()) myMap.force();
-      }
+      super.flush();
       if (myKeyHashToVirtualFileMapping != null && myKeyHashToVirtualFileMapping.isDirty()) {
         withLock(() -> myKeyHashToVirtualFileMapping.force());
       }
@@ -191,57 +113,34 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
 
   @Override
   public void close() throws StorageException {
+    super.close();
     try {
-      flush();
       if (myKeyHashToVirtualFileMapping != null) {
         withLock(() -> myKeyHashToVirtualFileMapping.close());
       }
-      myMap.close();
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
     }
     catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
+      unwrapCauseAndRethrow(e);
     }
   }
 
   @Override
   public void clear() throws StorageException{
     try {
-      myMap.close();
       if (myKeyHashToVirtualFileMapping != null) {
         withLock(() -> myKeyHashToVirtualFileMapping.close());
       }
     }
-    catch (IOException|RuntimeException e) {
+    catch (RuntimeException e) {
       LOG.error(e);
     }
     try {
-      IOUtil.deleteAllFilesStartingWith(getStorageFile());
       if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile());
-      initMapAndCache();
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
     }
     catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
+      unwrapCauseAndRethrow(e);
     }
+    super.clear();
   }
 
   @Override
@@ -308,14 +207,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
       throw new StorageException(e);
     }
     catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
+      return unwrapCauseAndRethrow(e);
     }
     finally {
       l.unlock();
@@ -385,7 +277,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
   private static File getSessionDir() {
     File sessionDirectory = mySessionDirectory;
     if (sessionDirectory == null) {
-      synchronized (MapIndexStorage.class) {
+      synchronized (VfsAwareMapIndexStorage.class) {
         sessionDirectory = mySessionDirectory;
         if (sessionDirectory == null) {
           try {
@@ -406,36 +298,6 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
     return new File(getSessionDir(), getProjectFile().getName() + "." + project.hashCode() + "." + id + "." + scope.isSearchInLibraries());
   }
 
-  @NotNull
-  @Override
-  public Collection<Key> getKeys() throws StorageException {
-    List<Key> keys = new ArrayList<>();
-    processKeys(Processors.cancelableCollectProcessor(keys), null, null);
-    return keys;
-  }
-
-  @Override
-  @NotNull
-  public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
-    l.lock();
-    try {
-      return myCache.get(key);
-    }
-    catch (RuntimeException e) {
-      final Throwable cause = e.getCause();
-      if (cause instanceof IOException) {
-        throw new StorageException(cause);
-      }
-      if (cause instanceof StorageException) {
-        throw (StorageException)cause;
-      }
-      throw e;
-    }
-    finally {
-      l.unlock();
-    }
-  }
-
   @Override
   public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
     try {
@@ -447,41 +309,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
           myLastScannedId = 0;
         }
       }
-
-      myMap.markDirty();
-      if (!myKeyIsUniqueForIndexedFile) {
-        read(key).addValue(inputId, value);
-        return;
-      }
-
-      ChangeTrackingValueContainer<Value> cached;
-      try {
-        l.lock();
-        cached = myCache.getIfCached(key);
-      } finally {
-        l.unlock();
-      }
-
-      if (cached != null) {
-        cached.addValue(inputId, value);
-        return;
-      }
-      // do not pollute the cache with keys unique to indexed file
-      ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<>(null);
-      valueContainer.addValue(inputId, value);
-      myMap.put(key, valueContainer);
-    }
-    catch (IOException e) {
-      throw new StorageException(e);
-    }
-  }
-
-  @Override
-  public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
-    try {
-      myMap.markDirty();
-      // important: assuming the key exists in the index
-      read(key).removeAssociatedValue(inputId);
+      super.addValue(key, inputId, value);
     }
     catch (IOException e) {
       throw new StorageException(e);

platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java

@@ -0,0 +1,260 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.application.Application;
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.ModalityState;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.openapi.util.UserDataHolder;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.impl.cache.impl.id.IdIndex;
+import com.intellij.psi.search.GlobalSearchScope;
+import com.intellij.util.Processor;
+import com.intellij.util.indexing.impl.*;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.EnumeratorIntegerDescriptor;
+import com.intellij.util.io.PersistentHashMap;
+import gnu.trove.TIntObjectHashMap;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+
+/**
+ * @author Eugene Zhuravlev
+ *         Date: Dec 10, 2007
+ */
+public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key, Value, Input>{
+  private static final Logger LOG = Logger.getInstance(VfsAwareMapReduceIndex.class);
+
+  static {
+    if (!DebugAssertions.DEBUG) {
+      final Application app = ApplicationManager.getApplication();
+      DebugAssertions.DEBUG = app.isEAP() || app.isInternal();
+    }
+  }
+
+  private final AtomicBoolean myInMemoryMode = new AtomicBoolean();
+  private final TIntObjectHashMap<Collection<Key>> myInMemoryKeys = new TIntObjectHashMap<Collection<Key>>();
+  private final SnapshotInputMappings<Key, Value, Input> mySnapshotInputMappings;
+
+  public VfsAwareMapReduceIndex(@NotNull IndexExtension<Key, Value, Input> extension,
+                                @NotNull IndexStorage<Key, Value> storage) throws IOException {
+    super(extension, storage, getForwardIndex(extension));
+    SharedIndicesData.registerIndex(myIndexId, extension);
+    mySnapshotInputMappings = myForwardIndex == null ?
+                              new SnapshotInputMappings<>(extension) :
+                              null;
+    installMemoryModeListener();
+  }
+
+  @TestOnly
+  public VfsAwareMapReduceIndex(@NotNull IndexExtension<Key, Value, Input> extension,
+                                @NotNull IndexStorage<Key, Value> storage,
+                                @NotNull ForwardIndex<Key, Value> forwardIndex) throws IOException {
+    super(extension, storage, forwardIndex);
+    SharedIndicesData.registerIndex(myIndexId, extension);
+    mySnapshotInputMappings = myForwardIndex == null ?
+                              new SnapshotInputMappings<>(extension) :
+                              null;
+    installMemoryModeListener();
+  }
+
+  @NotNull
+  @Override
+  protected UpdateData<Key, Value> calculateUpdateData(int inputId, @Nullable Input content) {
+    Map<Key, Value> data;
+    int hashId;
+    final boolean isContentPhysical = isContentPhysical(content);
+    if (mySnapshotInputMappings != null && content != null && isContentPhysical) {
+      final SnapshotInputMappings.Snapshot<Key, Value> snapshot = mySnapshotInputMappings.readPersistentDataOrMap(content);
+      data = snapshot.getData();
+      hashId = snapshot.getHashId();
+    } else {
+      data = mapInput(content);
+      hashId = 0;
+    }
+    return createUpdateData(data, () -> {
+      if (mySnapshotInputMappings != null && isContentPhysical) {
+        return new MapInputKeyIterator<>(mySnapshotInputMappings.readInputKeys(inputId));
+      }
+      if (myInMemoryMode.get()) {
+        synchronized (myInMemoryKeys) {
+          Collection<Key> keys = myInMemoryKeys.get(inputId);
+          if (keys != null) {
+            return new CollectionInputKeyIterator<>(keys);
+          }
+        }
+      }
+      if (myForwardIndex != null) {
+        return readInputKeys(inputId);
+      }
+      return EmptyInputKeyIterator.getInstance();
+    }, () -> {
+      if (myInMemoryMode.get()) {
+        synchronized (myInMemoryKeys) {
+          myInMemoryKeys.put(inputId, data.keySet());
+        }
+      } else {
+        if (mySnapshotInputMappings != null ) {
+          mySnapshotInputMappings.putInputHash(inputId, hashId);
+        } else {
+          myForwardIndex.putInputData(inputId, data);
+        }
+      }
+    });
+  }
+
+  @Override
+  public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
+    IndexingStamp.setFileIndexedStateCurrent(fileId, myIndexId);
+  }
+
+  @Override
+  public void resetIndexedStateForFile(int fileId) {
+    IndexingStamp.setFileIndexedStateOutdated(fileId, myIndexId);
+  }
+
+  @Override
+  public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
+    return IndexingStamp.isFileIndexedStateCurrent(fileId, myIndexId);
+  }
+
+  @Override
+  public boolean processAllKeys(@NotNull Processor<Key> processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
+    final Lock lock = getReadLock();
+    try {
+      lock.lock();
+      return ((VfsAwareIndexStorage<Key, Value>)myStorage).processKeys(processor, scope, idFilter);
+    }
+    finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public void checkCanceled() {
+    ProgressManager.checkCanceled();
+  }
+
+  @Override
+  protected void requestRebuild(@NotNull Exception ex) {
+    Runnable action = () -> FileBasedIndex.getInstance().requestRebuild(myIndexId, ex);
+    Application application = ApplicationManager.getApplication();
+    if (application.isUnitTestMode() || application.isHeadlessEnvironment()) {
+      // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask
+      application.invokeLater(action, ModalityState.any());
+    } else {
+      action.run();
+    }
+  }
+
+  @Override
+  public void clear() throws StorageException {
+    super.clear();
+    if (mySnapshotInputMappings != null) try {
+      mySnapshotInputMappings.clear();
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
+  }
+
+  @Override
+  public void flush() throws StorageException {
+    super.flush();
+    if (mySnapshotInputMappings != null) mySnapshotInputMappings.flush();
+  }
+
+  @Override
+  public void dispose() {
+    super.dispose();
+    if (mySnapshotInputMappings != null) try {
+      mySnapshotInputMappings.close();
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
+  }
+
+  @Nullable
+  private static <Key, Value> ForwardIndex<Key, Value> getForwardIndex(@NotNull IndexExtension<Key, Value, ?> indexExtension)
+    throws IOException {
+    final boolean hasSnapshotMapping = indexExtension instanceof FileBasedIndexExtension &&
+                                       ((FileBasedIndexExtension<Key, Value>)indexExtension).hasSnapshotMapping() &&
+                                       IdIndex.ourSnapshotMappingsEnabled;
+
+    return hasSnapshotMapping ? null : new SharedMapBasedForwardIndex<>(new MyForwardIndex<>(indexExtension));
+  }
+
+  private static class MyForwardIndex<Key, Value> extends MapBasedForwardIndex<Key, Value> {
+    protected MyForwardIndex(IndexExtension<Key, Value, ?> indexExtension) throws IOException {
+      super(indexExtension);
+    }
+
+    @NotNull
+    @Override
+    public PersistentHashMap<Integer, Collection<Key>> createMap() throws IOException {
+      return createIdToDataKeysIndex(myIndexExtension);
+    }
+
+    @NotNull
+    private static <K> PersistentHashMap<Integer, Collection<K>> createIdToDataKeysIndex(@NotNull IndexExtension<K, ?, ?> extension) throws IOException {
+      final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(extension.getName());
+      return new PersistentHashMap<>(indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, createInputsIndexExternalizer(extension));
+    }
+  }
+
+  private boolean isContentPhysical(Input content) {
+    return content == null ||
+           (content instanceof UserDataHolder &&
+            FileBasedIndexImpl.ourPhysicalContentKey.get((UserDataHolder)content, Boolean.FALSE));
+  }
+
+  private void installMemoryModeListener() {
+    IndexStorage<Key, Value> storage = getStorage();
+    if (storage instanceof MemoryIndexStorage) {
+      ((MemoryIndexStorage)storage).addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() {
+        @Override
+        public void bufferingStateChanged(boolean newState) {
+          myInMemoryMode.set(newState);
+        }
+
+        @Override
+        public void memoryStorageCleared() {
+          synchronized (myInMemoryKeys) {
+            myInMemoryKeys.clear();
+          }
+        }
+      });
+    }
+  }
+
+  protected static <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension) {
+    return extension instanceof CustomInputsIndexFileBasedIndexExtension
+           ? ((CustomInputsIndexFileBasedIndexExtension<K>)extension).createExternalizer()
+           : new InputIndexDataExternalizer<>(extension.getKeyDescriptor(), extension.getName());
+  }
+}

platform/util/src/com/intellij/util/indexing/InvertedIndexUtil.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.util.Condition;
+import com.intellij.util.containers.EmptyIntHashSet;
+import gnu.trove.TIntHashSet;
+import gnu.trove.TIntProcedure;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+
+@ApiStatus.Experimental
+public class InvertedIndexUtil {
+  @NotNull
+  public static <K, V, I> TIntHashSet collectInputIdsContainingAllKeys(@NotNull InvertedIndex<K, V, I> index,
+                                                                       @NotNull Collection<K> dataKeys,
+                                                                       @Nullable Condition<K> keyChecker,
+                                                                       @Nullable Condition<V> valueChecker,
+                                                                       @Nullable ValueContainer.IntPredicate idChecker)
+    throws StorageException {
+    TIntHashSet mainIntersection = null;
+
+    for (K dataKey : dataKeys) {
+      if (keyChecker != null && !keyChecker.value(dataKey)) continue;
+
+      final TIntHashSet copy = new TIntHashSet();
+      final ValueContainer<V> container = index.getData(dataKey);
+
+      for (ValueContainer.ValueIterator<V> valueIt = container.getValueIterator(); valueIt.hasNext(); ) {
+        final V value = valueIt.next();
+        if (valueChecker != null && !valueChecker.value(value)) {
+          continue;
+        }
+
+        ValueContainer.IntIterator iterator = valueIt.getInputIdsIterator();
+
+        final ValueContainer.IntPredicate predicate;
+        if (mainIntersection == null || iterator.size() < mainIntersection.size() || (predicate = valueIt.getValueAssociationPredicate()) == null) {
+          while (iterator.hasNext()) {
+            final int id = iterator.next();
+            if (mainIntersection == null && (idChecker == null || idChecker.contains(id)) ||
+                mainIntersection != null && mainIntersection.contains(id)
+              ) {
+              copy.add(id);
+            }
+          }
+        }
+        else {
+          mainIntersection.forEach(new TIntProcedure() {
+            @Override
+            public boolean execute(int id) {
+              if (predicate.contains(id)) copy.add(id);
+              return true;
+            }
+          });
+        }
+      }
+
+      mainIntersection = copy;
+      if (mainIntersection.isEmpty()) {
+        return EmptyIntHashSet.INSTANCE;
+      }
+    }
+
+    return mainIntersection == null ? EmptyIntHashSet.INSTANCE : mainIntersection;
+  }
+}

platform/util/src/com/intellij/util/indexing/ValueContainer.java

@@ -17,6 +17,7 @@
 package com.intellij.util.indexing;
 
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.Iterator;
 
@@ -33,12 +34,19 @@ public abstract class ValueContainer<Value> {
     int size();
   }
 
+  public interface IntPredicate {
+    boolean contains(int id);
+  }
+
   @NotNull
   public abstract ValueIterator<Value> getValueIterator();
 
   public interface ValueIterator<Value> extends Iterator<Value> {
     @NotNull
     IntIterator getInputIdsIterator();
+
+    @Nullable
+    IntPredicate getValueAssociationPredicate();
   }
 
   public abstract int size();

platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,14 +15,13 @@
  */
 package com.intellij.util.indexing.containers;
 
-import com.intellij.util.indexing.DebugAssertions;
-import com.intellij.util.indexing.IntPredicate;
+import com.intellij.util.indexing.impl.DebugAssertions;
 import com.intellij.util.indexing.ValueContainer;
 import gnu.trove.TIntProcedure;
 
 import java.util.Arrays;
 
-import static com.intellij.util.indexing.DebugAssertions.EXTRA_SANITY_CHECKS;
+import static com.intellij.util.indexing.impl.DebugAssertions.EXTRA_SANITY_CHECKS;
 
 /**
  * Class buffers changes in 2 modes:
@@ -267,10 +266,10 @@ public class ChangeBufferingList implements Cloneable {
     return intContainer.size() == 0;
   }
 
-  public IntPredicate intPredicate() {
-    final IntPredicate predicate = getRandomAccessContainer().intPredicate();
+  public ValueContainer.IntPredicate intPredicate() {
+    final ValueContainer.IntPredicate predicate = getRandomAccessContainer().intPredicate();
     if (checkSet != null) {
-      return new IntPredicate() {
+      return new ValueContainer.IntPredicate() {
         @Override
         public boolean contains(int id) {
           boolean answer = predicate.contains(id);

platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
 package com.intellij.util.indexing.containers;
 
 import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.util.indexing.IntPredicate;
 import com.intellij.util.indexing.ValueContainer;
 
 /**
@@ -127,8 +126,8 @@ class IdBitSet implements Cloneable, RandomAccessIntContainer {
   }
 
   @Override
-  public IntPredicate intPredicate() {
-    return new IntPredicate() {
+  public ValueContainer.IntPredicate intPredicate() {
+    return new ValueContainer.IntPredicate() {
       @Override
       public boolean contains(int id) {
         return IdBitSet.this.contains(id);

platform/util/src/com/intellij/util/indexing/containers/IdSet.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
  */
 package com.intellij.util.indexing.containers;
 
-import com.intellij.util.indexing.IntPredicate;
+import com.intellij.util.indexing.ValueContainer;
 import gnu.trove.TIntHashSet;
 
 public class IdSet extends TIntHashSet implements RandomAccessIntContainer {
@@ -36,8 +36,8 @@ public class IdSet extends TIntHashSet implements RandomAccessIntContainer {
   }
 
   @Override
-  public IntPredicate intPredicate() {
-    return new IntPredicate() {
+  public ValueContainer.IntPredicate intPredicate() {
+    return new ValueContainer.IntPredicate() {
       @Override
       public boolean contains(int id) {
         return IdSet.this.contains(id);

platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
  */
 package com.intellij.util.indexing.containers;
 
-import com.intellij.util.indexing.IntPredicate;
+import com.intellij.util.indexing.ValueContainer;
 
 /**
 * Created by Maxim.Mossienko on 5/27/2014.
@@ -25,7 +25,7 @@ interface RandomAccessIntContainer {
   boolean add(int value);
   boolean remove(int value);
   IntIdsIterator intIterator();
-  IntPredicate intPredicate();
+  ValueContainer.IntPredicate intPredicate();
   void compact();
   int size();
 

platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
  */
 package com.intellij.util.indexing.containers;
 
-import com.intellij.util.indexing.IntPredicate;
+import com.intellij.util.indexing.ValueContainer;
 import gnu.trove.TIntProcedure;
 
 /**
@@ -94,8 +94,8 @@ public class SortedIdSet implements Cloneable, RandomAccessIntContainer {
   }
 
   @Override
-  public IntPredicate intPredicate() {
-    return new IntPredicate() {
+  public ValueContainer.IntPredicate intPredicate() {
+    return new ValueContainer.IntPredicate() {
 
       @Override
       public boolean contains(int id) {

platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.IndexExtension;
+import com.intellij.util.io.KeyDescriptor;
+import org.jetbrains.annotations.NotNull;
+
+public abstract class AbstractForwardIndex<Key, Value> implements ForwardIndex<Key,Value> {
+  protected final ID<Key, Value> myIndexId;
+  protected final KeyDescriptor<Key> myKeyDescriptor;
+  protected final IndexExtension<Key, Value, ?> myIndexExtension;
+
+  protected AbstractForwardIndex(@NotNull IndexExtension<Key, Value, ?> extension) {
+    myIndexId = extension.getName();
+    myKeyDescriptor = extension.getKeyDescriptor();
+    myIndexExtension = extension;
+  }
+
+  @NotNull
+  public IndexExtension<Key, Value, ?> getIndexExtension() {
+    return myIndexExtension;
+  }
+
+  public boolean hasOnlyKeysData() {
+    return true;
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/ChangeTrackingValueContainer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
 import com.intellij.openapi.util.Computable;
+import com.intellij.util.indexing.ValueContainer;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.DataInputOutputUtil;
 import gnu.trove.TIntHashSet;
@@ -31,7 +32,7 @@ import java.io.IOException;
  * @author Eugene Zhuravlev
  *         Date: Dec 20, 2007
  */
-class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>{
+public class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>{
   // there is no volatile as we modify under write lock and read under read lock
   private ValueContainerImpl<Value> myAdded;
   private TIntHashSet myInvalidated;
@@ -53,7 +54,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
       merged.addValue(inputId, value);
     }
 
-    if (myAdded == null) myAdded = new ValueContainerImpl<>();
+    if (myAdded == null) myAdded = new ValueContainerImpl<Value>();
     myAdded.addValue(inputId, value);
   }
 
@@ -77,7 +78,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
 
   @NotNull
   @Override
-  public ValueIterator<Value> getValueIterator() {
+  public ValueContainer.ValueIterator<Value> getValueIterator() {
     return getMergedData().getValueIterator();
   }
 
@@ -112,7 +113,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
           (newMerged.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD ||
            (myAdded != null && myAdded.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD))) {
         // Calculate file ids that have Value mapped to avoid O(NumberOfValuesInMerged) during removal
-        fileId2ValueMapping = new FileId2ValueMapping<>(newMerged);
+        fileId2ValueMapping = new FileId2ValueMapping<Value>(newMerged);
       }
       final FileId2ValueMapping<Value> finalFileId2ValueMapping = fileId2ValueMapping;
       if (myInvalidated != null) {
@@ -132,7 +133,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
           fileId2ValueMapping.disableOneValuePerFileValidation();
         }
 
-        myAdded.forEach(new ContainerAction<Value>() {
+        myAdded.forEach(new ValueContainer.ContainerAction<Value>() {
           @Override
           public boolean perform(final int inputId, final Value value) {
             // enforcing "one-value-per-file for particular key" invariant

platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+
+public class CollectionInputKeyIterator<Key, Value> implements ForwardIndex.InputKeyIterator<Key, Value> {
+  private final Collection<Key> mySeq;
+  private Iterator<Key> myIt;
+
+  public CollectionInputKeyIterator(Collection<Key> seq) {
+    mySeq = seq;
+  }
+
+  @Override
+  public boolean isAssociatedValueEqual(@Nullable Value value) {
+    return false;
+  }
+
+  @Override
+  public boolean hasNext() {
+    init();
+    return myIt.hasNext();
+  }
+
+  @Override
+  public Key next() {
+    return myIt.next();
+  }
+
+  @Override
+  public void remove() {
+    throw new UnsupportedOperationException();
+  }
+
+  public Collection<Key> getCollection() {
+    return mySeq == null ? Collections.<Key>emptySet() : mySeq;
+  }
+
+  private void init() {
+    if (myIt == null) {
+      myIt = getCollection().iterator();
+    }
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/DebugAssertions.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,9 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
-import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.util.SystemProperties;
 import com.intellij.util.containers.hash.LinkedHashMap;
@@ -27,9 +26,10 @@ import java.util.Formatter;
 public class DebugAssertions {
   private static final Logger LOG = Logger.getInstance(DebugAssertions.class);
 
-  public static final boolean DEBUG = SystemProperties.getBooleanProperty(
+  @SuppressWarnings("StaticNonFinalField")
+  public static volatile boolean DEBUG = SystemProperties.getBooleanProperty(
     "intellij.idea.indices.debug",
-    ApplicationManager.getApplication().isInternal() || ApplicationManager.getApplication().isEAP()
+    false
   );
 
   public static final boolean EXTRA_SANITY_CHECKS = SystemProperties.getBooleanProperty(
@@ -53,12 +53,12 @@ public class DebugAssertions {
     LOG.error(new Formatter().format(message, args));
   }
 
-  static <Key> boolean equals(Collection<Key> keys, Collection<Key> keys2, KeyDescriptor<Key> keyDescriptor) {
+  public static <Key> boolean equals(Collection<Key> keys, Collection<Key> keys2, KeyDescriptor<Key> keyDescriptor) {
     if (keys == null && keys2 == null) return true;
     if (keys == null || keys2 == null || keys.size() != keys2.size()) return false;
-    LinkedHashMap<Key, Boolean> map = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor);
+    LinkedHashMap<Key, Boolean> map = new LinkedHashMap<Key, Boolean>(keys.size(), 0.8f, keyDescriptor);
     for(Key key:keys) map.put(key, Boolean.TRUE);
-    LinkedHashMap<Key, Boolean> map2 = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor);
+    LinkedHashMap<Key, Boolean> map2 = new LinkedHashMap<Key, Boolean>(keys.size(), 0.8f, keyDescriptor);
     for(Key key:keys2) map2.put(key, Boolean.TRUE);
     return map.equals(map2);
   }

platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.ThrowableComputable;
+import com.intellij.util.SystemProperties;
+import com.intellij.util.ThrowableRunnable;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.StorageException;
+import gnu.trove.THashSet;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class DiffUpdateData<Key, Value> extends UpdateData<Key,Value> {
+  public static final boolean ourDiffUpdateEnabled = SystemProperties.getBooleanProperty("idea.disable.diff.index.update", true);
+
+  public DiffUpdateData(@NotNull Map<Key, Value> newData,
+                        @NotNull ThrowableComputable<ForwardIndex.InputKeyIterator<Key, Value>, IOException> currentData,
+                        @NotNull ID<Key, Value> indexId, ThrowableRunnable<IOException> forwardIndexUpdate) {
+    super(newData, currentData, indexId, forwardIndexUpdate);
+  }
+
+  @Override
+  public void iterateKeys(int inputId,
+                          KeyValueUpdateProcessor<Key, Value> addProcessor,
+                          KeyValueUpdateProcessor<Key, Value> updateProcessor,
+                          RemovedKeyProcessor<Key> removeProcessor) throws StorageException {
+    final Set<Key> processedKeys = new THashSet<Key>();
+    int oldSize = 0; //kept for debug reasons
+    int addedKeys = 0;
+    int removedKeys = 0;
+    boolean newDataIsEmpty = myNewData.isEmpty();
+    final ForwardIndex.InputKeyIterator<Key, Value> currentData;
+    try {
+      currentData = myCurrentData.compute();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    while (currentData.hasNext()) {
+      oldSize++;
+      Key key = currentData.next();
+      if (!newDataIsEmpty) {
+        processedKeys.add(key);
+      }
+      if (newDataIsEmpty || !myNewData.containsKey(key)) {
+        removeProcessor.process(key, inputId);
+        removedKeys++;
+      } else {
+        Value newValue = myNewData.get(key);
+        if (!currentData.isAssociatedValueEqual(newValue)) {
+          updateProcessor.process(key, newValue, inputId);
+          removedKeys++;
+          addedKeys++;
+        }
+      }
+    }
+
+    if (!newDataIsEmpty) {
+      for (Map.Entry<Key, Value> entry : myNewData.entrySet()) {
+        if (!processedKeys.contains(entry.getKey())) {
+          addProcessor.process(entry.getKey(), entry.getValue(), inputId);
+          addedKeys++;
+        }
+      }
+    }
+
+    int totalRequests = requests.incrementAndGet();
+    totalRemovals.addAndGet(oldSize);
+    totalAdditions.addAndGet(myNewData.size());
+    incrementalAdditions.addAndGet(removedKeys);
+    incrementalRemovals.addAndGet(addedKeys);
+
+    if ((totalRequests & 0xFFF) == 0 && DebugAssertions.DEBUG) {
+      Logger.getInstance(getClass()).info("Incremental index diff update:"+requests +
+                                          ", removals:" + totalRemovals + "->" + incrementalRemovals +
+                                          ", additions:" +totalAdditions + "->" +incrementalAdditions);
+    }
+  }
+
+  private static final AtomicInteger requests = new AtomicInteger();
+  private static final AtomicInteger totalRemovals = new AtomicInteger();
+  private static final AtomicInteger totalAdditions = new AtomicInteger();
+  private static final AtomicInteger incrementalRemovals = new AtomicInteger();
+  private static final AtomicInteger incrementalAdditions = new AtomicInteger();
+
+  @NotNull
+  protected Map<Key, Value> getMap() {
+    return myNewData;
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.Nullable;
+
+@ApiStatus.Experimental
+public class EmptyInputKeyIterator<Key, Value> implements ForwardIndex.InputKeyIterator<Key,Value> {
+  public static final EmptyInputKeyIterator EMPTY_INPUT_KEY_ITERATOR = new EmptyInputKeyIterator();
+
+  public static <Key, Value> ForwardIndex.InputKeyIterator<Key, Value> getInstance() {
+    //noinspection unchecked
+    return EMPTY_INPUT_KEY_ITERATOR;
+  }
+
+  @Override
+  public boolean isAssociatedValueEqual(@Nullable Value value) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean hasNext() {
+    return false;
+  }
+
+  @Override
+  public Key next() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void remove() {
+    throw new UnsupportedOperationException();
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/FileId2ValueMapping.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,9 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
 import com.intellij.util.SmartList;
+import com.intellij.util.indexing.ValueContainer;
 import gnu.trove.TIntArrayList;
 import gnu.trove.TIntObjectHashMap;
 
@@ -30,7 +31,7 @@ class FileId2ValueMapping<Value> {
   private boolean myOnePerFileValidationEnabled = true;
 
   FileId2ValueMapping(ValueContainerImpl<Value> _valueContainer) {
-    id2ValueMap = new TIntObjectHashMap<>();
+    id2ValueMap = new TIntObjectHashMap<Value>();
     valueContainer = _valueContainer;
 
     TIntArrayList removedFileIdList = null;
@@ -45,7 +46,7 @@ class FileId2ValueMapping<Value> {
         if (previousValue != null) {  // delay removal of duplicated id -> value mapping since it will affect valueIterator we are using
           if (removedFileIdList == null) {
             removedFileIdList = new TIntArrayList();
-            removedValueList = new SmartList<>();
+            removedValueList = new SmartList<Value>();
           }
           removedFileIdList.add(id);
           removedValueList.add(previousValue);

platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+@ApiStatus.Experimental
+public interface ForwardIndex<Key, Value> {
+  @NotNull
+  InputKeyIterator<Key, Value> getInputKeys(int inputId) throws IOException;
+
+  void putInputData(int inputId, @NotNull Map<Key, Value> data) throws IOException;
+
+  void flush();
+
+  void clear() throws IOException;
+
+  void close() throws IOException;
+
+  @ApiStatus.Experimental
+  interface InputKeyIterator<Key, Value> extends Iterator<Key> {
+    boolean isAssociatedValueEqual(@Nullable Value value);
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/IndexStorage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,21 +14,21 @@
  * limitations under the License.
  */
 
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
-import com.intellij.psi.search.GlobalSearchScope;
-import com.intellij.util.Processor;
+import com.intellij.util.indexing.StorageException;
+import com.intellij.util.indexing.ValueContainer;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
 import java.io.Flushable;
 import java.io.IOException;
-import java.util.Collection;
 
 /**
  * @author Eugene Zhuravlev
  *         Date: Dec 10, 2007
  */
+@ApiStatus.Experimental
 public interface IndexStorage<Key, Value> extends Flushable {
   
   void addValue(Key key, int inputId, Value value) throws StorageException;
@@ -40,10 +40,7 @@ public interface IndexStorage<Key, Value> extends Flushable {
   @NotNull
   ValueContainer<Value> read(Key key) throws StorageException;
 
-  boolean processKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
-
-  @NotNull
-  Collection<Key> getKeys() throws StorageException;
+  void clearCaches();
 
   void close() throws StorageException;
 

platform/util/src/com/intellij/util/indexing/impl/InputIndexDataExternalizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,8 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
+import com.intellij.util.indexing.ID;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.DataInputOutputUtil;
 import com.intellij.util.io.KeyDescriptor;
@@ -57,7 +58,7 @@ public class InputIndexDataExternalizer<K> implements DataExternalizer<Collectio
   public Collection<K> read(@NotNull DataInput in) throws IOException {
     try {
       final int size = DataInputOutputUtil.readINT(in);
-      final List<K> list = new ArrayList<>(size);
+      final List<K> list = new ArrayList<K>(size);
       for (int idx = 0; idx < size; idx++) {
         list.add(myKeyDescriptor.read(in));
       }

platform/util/src/com/intellij/util/indexing/impl/InvertedIndexValueIterator.java

@@ -13,16 +13,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
+import com.intellij.util.indexing.ValueContainer;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
 
 /**
  * Created by Maxim.Mossienko on 11/22/2016.
  */
-interface InvertedIndexValueIterator<Value> extends ValueContainer.ValueIterator<Value> {
+@ApiStatus.Experimental
+public interface InvertedIndexValueIterator<Value> extends ValueContainer.ValueIterator<Value> {
+  @Override
   @NotNull
-  IntPredicate getValueAssociationPredicate();
+  ValueContainer.IntPredicate getValueAssociationPredicate();
 
   Object getFileSetObject();
 }

platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.util.indexing.IndexExtension;
+import com.intellij.util.io.IOUtil;
+import com.intellij.util.io.PersistentHashMap;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+public abstract class MapBasedForwardIndex<Key, Value> extends AbstractForwardIndex<Key,Value> {
+  @NotNull
+  private volatile PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
+
+  protected MapBasedForwardIndex(IndexExtension<Key, Value, ?> indexExtension) throws IOException {
+    super(indexExtension);
+    myInputsIndex = createMap();
+  }
+
+  @NotNull
+  public abstract PersistentHashMap<Integer, Collection<Key>> createMap() throws IOException;
+
+  @NotNull
+  @Override
+  public InputKeyIterator<Key, Value> getInputKeys(final int inputId) throws IOException {
+    return new CollectionInputKeyIterator<Key, Value>(myInputsIndex.get(inputId));
+  }
+
+  @NotNull
+  public PersistentHashMap<Integer, Collection<Key>> getInputsIndex() {
+    return myInputsIndex;
+  }
+
+  @Override
+  public void putInputData(int inputId, @NotNull Map<Key, Value> data) throws IOException {
+    putData(inputId, data.keySet());
+  }
+
+  public void putData(int inputId, Collection<Key> keyCollection) throws IOException {
+    if (keyCollection.size() > 0) {
+      myInputsIndex.put(inputId, keyCollection);
+    }
+    else {
+      myInputsIndex.remove(inputId);
+    }
+  }
+
+  @Override
+  public void flush() {
+    if (myInputsIndex.isDirty()) {
+      myInputsIndex.force();
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    myInputsIndex.close();
+  }
+
+  @Override
+  public void clear() throws IOException {
+    final File baseFile = myInputsIndex.getBaseFile();
+    try {
+      myInputsIndex.close();
+    }
+    catch (Throwable ignored) {
+    }
+    if (baseFile != null) {
+      IOUtil.deleteAllFilesStartingWith(baseFile);
+    }
+    myInputsIndex = createMap();
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/MapIndexStorage.java

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.util.Processor;
+import com.intellij.util.containers.SLRUCache;
+import com.intellij.util.indexing.StorageException;
+import com.intellij.util.indexing.ValueContainer;
+import com.intellij.util.io.*;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public abstract class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value> {
+  private static final Logger LOG = Logger.getInstance(MapIndexStorage.class);
+  protected PersistentMap<Key, UpdatableValueContainer<Value>> myMap;
+  protected SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
+  protected final File myBaseStorageFile;
+  protected final KeyDescriptor<Key> myKeyDescriptor;
+  private final int myCacheSize;
+
+  protected final Lock l = new ReentrantLock();
+  private final DataExternalizer<Value> myDataExternalizer;
+  private final boolean myKeyIsUniqueForIndexedFile;
+
+  public MapIndexStorage(@NotNull File storageFile,
+                         @NotNull KeyDescriptor<Key> keyDescriptor,
+                         @NotNull DataExternalizer<Value> valueExternalizer,
+                         final int cacheSize,
+                         boolean keyIsUniqueForIndexedFile) throws IOException {
+    this(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, true);
+  }
+
+  protected MapIndexStorage(@NotNull File storageFile,
+                            @NotNull KeyDescriptor<Key> keyDescriptor,
+                            @NotNull DataExternalizer<Value> valueExternalizer,
+                            final int cacheSize,
+                            boolean keyIsUniqueForIndexedFile,
+                            boolean initialize) throws IOException {
+    myBaseStorageFile = storageFile;
+    myKeyDescriptor = keyDescriptor;
+    myCacheSize = cacheSize;
+    myDataExternalizer = valueExternalizer;
+    myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
+    if (initialize) initMapAndCache();
+  }
+
+  protected void initMapAndCache() throws IOException {
+    final ValueContainerMap<Key, Value> map;
+    PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(
+      new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() {
+        @Override
+        public void checkCancellation() {
+          checkCanceled();
+        }
+      });
+    PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
+    try {
+      map = new ValueContainerMap<Key, Value>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
+    } finally {
+      PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
+      PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
+    }
+    myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
+      @Override
+      @NotNull
+      public ChangeTrackingValueContainer<Value> createValue(final Key key) {
+        return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
+          @NotNull
+          @Override
+          public Object getLock() {
+            return map.getDataAccessLock();
+          }
+
+          @Nullable
+          @Override
+          public ValueContainer<Value> compute() {
+            ValueContainer<Value> value;
+            try {
+              value = map.get(key);
+              if (value == null) {
+                value = new ValueContainerImpl<Value>();
+              }
+            }
+            catch (IOException e) {
+              throw new RuntimeException(e);
+            }
+            return value;
+          }
+        });
+      }
+
+      @Override
+      protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
+        if (valueContainer.isDirty()) {
+          try {
+            map.put(key, valueContainer);
+          }
+          catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        }
+      }
+    };
+
+    myMap = map;
+  }
+
+  protected abstract void checkCanceled();
+
+  @NotNull
+  private File getStorageFile() {
+    return new File(myBaseStorageFile.getPath() + ".storage");
+  }
+
+  @Override
+  public void flush() {
+    l.lock();
+    try {
+      if (!myMap.isClosed()) {
+        myCache.clear();
+        if (myMap.isDirty()) myMap.force();
+      }
+    }
+    finally {
+      l.unlock();
+    }
+  }
+
+  @Override
+  public void close() throws StorageException {
+    try {
+      flush();
+      myMap.close();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      unwrapCauseAndRethrow(e);
+    }
+  }
+
+  @Override
+  public void clear() throws StorageException{
+    try {
+      myMap.close();
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
+    catch (RuntimeException e) {
+      LOG.error(e);
+    }
+    try {
+      IOUtil.deleteAllFilesStartingWith(getStorageFile());
+      initMapAndCache();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      unwrapCauseAndRethrow(e);
+    }
+  }
+
+  @Override
+  @NotNull
+  public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
+    l.lock();
+    try {
+      return myCache.get(key);
+    }
+    catch (RuntimeException e) {
+      return unwrapCauseAndRethrow(e);
+    }
+    finally {
+      l.unlock();
+    }
+  }
+
+  @Override
+  public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
+    try {
+      myMap.markDirty();
+      if (!myKeyIsUniqueForIndexedFile) {
+        read(key).addValue(inputId, value);
+        return;
+      }
+
+      ChangeTrackingValueContainer<Value> cached;
+      try {
+        l.lock();
+        cached = myCache.getIfCached(key);
+      } finally {
+        l.unlock();
+      }
+
+      if (cached != null) {
+        cached.addValue(inputId, value);
+        return;
+      }
+      // do not pollute the cache with keys unique to indexed file
+      ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<Value>(null);
+      valueContainer.addValue(inputId, value);
+      myMap.put(key, valueContainer);
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+  }
+
+  @Override
+  public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
+    try {
+      myMap.markDirty();
+      // important: assuming the key exists in the index
+      read(key).removeAssociatedValue(inputId);
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+  }
+
+  @Override
+  public void clearCaches() {
+
+  }
+
+  protected static <T> T unwrapCauseAndRethrow(RuntimeException e) throws StorageException {
+    final Throwable cause = e.getCause();
+    if (cause instanceof IOException) {
+      throw new StorageException(cause);
+    }
+    if (cause instanceof StorageException) {
+      throw (StorageException)cause;
+    }
+    throw e;
+  }
+
+  @TestOnly
+  public boolean processKeys(@NotNull Processor<Key> processor) throws StorageException {
+    l.lock();
+    try {
+      myCache.clear(); // this will ensure that all new keys are made into the map
+      return myMap.processKeys(processor);
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      unwrapCauseAndRethrow(e);
+      return false;
+    }
+    finally {
+      l.unlock();
+    }
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.openapi.util.Comparing;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+
+public class MapInputKeyIterator<Key, Value> implements ForwardIndex.InputKeyIterator<Key,Value> {
+  private final Map<Key, Value> myMap;
+  private Iterator<Map.Entry<Key, Value>> myIterator;
+  private Value myCurrentValue;
+
+  public MapInputKeyIterator(Map<Key, Value> map) {
+    myMap = map;
+  }
+
+  @Override
+  public boolean isAssociatedValueEqual(@Nullable Value value) {
+    return Comparing.equal(myCurrentValue, value);
+  }
+
+  @Override
+  public boolean hasNext() {
+    init();
+    return myIterator.hasNext();
+  }
+
+  @Override
+  public Key next() {
+    Map.Entry<Key, Value> entry = myIterator.next();
+    myCurrentValue = entry.getValue();
+    return entry.getKey();
+  }
+
+  @Override
+  public void remove() {
+    throw new UnsupportedOperationException();
+  }
+
+  private void init() {
+    if (myIterator == null) {
+      myIterator = (myMap == null ? Collections.<Key, Value>emptyMap() : myMap).entrySet().iterator();
+    }
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java

@@ -0,0 +1,350 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.progress.ProcessCanceledException;
+import com.intellij.openapi.util.Comparing;
+import com.intellij.openapi.util.Computable;
+import com.intellij.openapi.util.LowMemoryWatcher;
+import com.intellij.openapi.util.ThrowableComputable;
+import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
+import com.intellij.util.ThrowableRunnable;
+import com.intellij.util.indexing.*;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.DataOutputStream;
+import com.intellij.util.io.UnsyncByteArrayInputStream;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+@ApiStatus.Experimental
+public abstract class MapReduceIndex<Key,Value, Input> implements InvertedIndex<Key, Value, Input> {
+  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.MapReduceIndex");
+  @NotNull protected final ID<Key, Value> myIndexId;
+  @NotNull protected final IndexStorage<Key, Value> myStorage;
+
+  protected final DataExternalizer<Value> myValueExternalizer;
+  protected final IndexExtension<Key, Value, Input> myExtension;
+  private final AtomicLong myModificationStamp = new AtomicLong();
+  private final DataIndexer<Key, Value, Input> myIndexer;
+
+  protected volatile ForwardIndex<Key, Value> myForwardIndex;
+  private final boolean myUseDiffUpdate;
+
+  private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
+  private volatile boolean myDisposed;
+
+  private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() {
+    @Override
+    public void run() {
+      try {
+        Lock writeLock = getWriteLock();
+        if (writeLock.tryLock()) {
+          try {
+            myStorage.clearCaches();
+          } finally {
+            writeLock.unlock();
+          }
+        }
+        flush();
+      } catch (StorageException e) {
+        LOG.info(e);
+        requestRebuild(e);
+      }
+    }
+  });
+
+  protected MapReduceIndex(@NotNull IndexExtension<Key, Value, Input> extension,
+                           @NotNull IndexStorage<Key, Value> storage,
+                           ForwardIndex<Key, Value> forwardIndex) throws IOException {
+    myIndexId = extension.getName();
+    myExtension = extension;
+    myIndexer = myExtension.getIndexer();
+    myStorage = storage;
+    myValueExternalizer = extension.getValueExternalizer();
+    myForwardIndex = forwardIndex;
+    myUseDiffUpdate = DiffUpdateData.ourDiffUpdateEnabled && (forwardIndex == null ||
+                                                              myForwardIndex instanceof AbstractForwardIndex &&
+                                                              !((AbstractForwardIndex)myForwardIndex).hasOnlyKeysData());
+  }
+
+  @NotNull
+  public IndexStorage<Key, Value> getStorage() {
+    return myStorage;
+  }
+
+  @Override
+  public void clear() throws StorageException {
+    try {
+      getWriteLock().lock();
+      myStorage.clear();
+      if (myForwardIndex != null) myForwardIndex.clear();
+    }
+    catch (StorageException e) {
+      LOG.error(e);
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
+    finally {
+      getWriteLock().unlock();
+    }
+  }
+
+  @Override
+  public void flush() throws StorageException{
+    try {
+      getReadLock().lock();
+      if (myForwardIndex != null) myForwardIndex.flush();
+      myStorage.flush();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    catch (RuntimeException e) {
+      final Throwable cause = e.getCause();
+      if (cause instanceof StorageException || cause instanceof IOException) {
+        throw new StorageException(cause);
+      }
+      else {
+        throw e;
+      }
+    }
+    finally {
+      getReadLock().unlock();
+    }
+  }
+
+  @Override
+  public void dispose() {
+    myLowMemoryFlusher.stop();
+    final Lock lock = getWriteLock();
+    try {
+      lock.lock();
+      try {
+        myStorage.close();
+      }
+      finally {
+        try {
+          if (myForwardIndex != null) myForwardIndex.close();
+        }
+        catch (IOException e) {
+          LOG.error(e);
+        }
+      }
+    }
+    catch (StorageException e) {
+      LOG.error(e);
+    }
+    finally {
+      myDisposed = true;
+      lock.unlock();
+    }
+  }
+
+  @NotNull
+  public final Lock getReadLock() {
+    return myLock.readLock();
+  }
+
+  @NotNull
+  public final Lock getWriteLock() {
+    return myLock.writeLock();
+  }
+
+  @Override
+  @NotNull
+  public ValueContainer<Value> getData(@NotNull final Key key) throws StorageException {
+    final Lock lock = getReadLock();
+    try {
+      lock.lock();
+      if (myDisposed) {
+        return new ValueContainerImpl<Value>();
+      }
+      ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
+      return myStorage.read(key);
+    }
+    finally {
+      ValueContainerImpl.ourDebugIndexInfo.set(null);
+      lock.unlock();
+    }
+  }
+
+  @NotNull
+  @Override
+  public final Computable<Boolean> update(final int inputId, @Nullable final Input content) {
+    final UpdateData<Key, Value> updateData = calculateUpdateData(inputId, content);
+
+    return new Computable<Boolean>() {
+      @Override
+      public Boolean compute() {
+        try {
+          updateWithMap(inputId, updateData);
+        }
+        catch (StorageException ex) {
+          LOG.info("Exception during updateWithMap:" + ex);
+          requestRebuild(ex);
+          return Boolean.FALSE;
+        }
+        catch (ProcessCanceledException ex) {
+          LOG.info("Exception during updateWithMap:" + ex);
+          requestRebuild(ex);
+          return Boolean.FALSE;
+        }
+
+        return Boolean.TRUE;
+      }
+    };
+  }
+
+  @NotNull
+  protected UpdateData<Key, Value> calculateUpdateData(final int inputId, @Nullable Input content) {
+    final Map<Key, Value> data = mapInput(content);
+    return createUpdateData(data, new ThrowableComputable<ForwardIndex.InputKeyIterator<Key, Value>, IOException>() {
+      @Override
+      public ForwardIndex.InputKeyIterator<Key, Value> compute() throws IOException {
+        return readInputKeys(inputId);
+      }
+    }, new ThrowableRunnable<IOException>() {
+      @Override
+      public void run() throws IOException {
+        myForwardIndex.putInputData(inputId, data);
+      }
+    });
+  }
+
+  @NotNull
+  protected ForwardIndex.InputKeyIterator<Key, Value> readInputKeys(int inputId) throws IOException {
+    return myForwardIndex.getInputKeys(inputId);
+  }
+
+  @NotNull
+  protected UpdateData<Key, Value> createUpdateData(Map<Key, Value> data,
+                                                    ThrowableComputable<ForwardIndex.InputKeyIterator<Key, Value>, IOException> keys,
+                                                    ThrowableRunnable<IOException> forwardIndexUpdate) {
+    return myUseDiffUpdate ? new DiffUpdateData<Key, Value>(data, keys, myIndexId, forwardIndexUpdate)
+                           : new SimpleUpdateData<Key, Value>(data, keys, myIndexId, forwardIndexUpdate);
+  }
+
+  protected Map<Key, Value> mapInput(Input content) {
+    if (content == null) {
+      return Collections.emptyMap();
+    }
+    else {
+      Map<Key, Value> data = myIndexer.map(content);
+      checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer);
+      checkCanceled();
+      return data;
+    }
+  }
+
+  public abstract void checkCanceled();
+
+  protected abstract void requestRebuild(Exception e);
+
+  public long getModificationStamp() {
+    return myModificationStamp.get();
+  }
+
+  private final UpdateData.RemovedKeyProcessor<Key>
+    myRemovedKeyProcessor = new UpdateData.RemovedKeyProcessor<Key>() {
+    @Override
+    public void process(Key key, int inputId) throws StorageException {
+      myModificationStamp.incrementAndGet();
+      myStorage.removeAllValues(key, inputId);
+    }
+  };
+
+  private final UpdateData.KeyValueUpdateProcessor<Key, Value> myAddedKeyProcessor = new UpdateData.KeyValueUpdateProcessor<Key, Value>() {
+    @Override
+    public void process(Key key, Value value, int inputId) throws StorageException {
+      myModificationStamp.incrementAndGet();
+      myStorage.addValue(key, inputId, value);
+    }
+  };
+
+  private final UpdateData.KeyValueUpdateProcessor<Key, Value> myUpdatedKeyProcessor = new UpdateData.KeyValueUpdateProcessor<Key, Value>() {
+    @Override
+    public void process(Key key, Value value, int inputId) throws StorageException {
+      myModificationStamp.incrementAndGet();
+      myStorage.removeAllValues(key, inputId);
+      myStorage.addValue(key, inputId, value);
+    }
+  };
+
+  protected void updateWithMap(final int inputId,
+                               @NotNull UpdateData<Key, Value> updateData) throws StorageException {
+    getWriteLock().lock();
+    try {
+      try {
+        ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
+        updateData.iterateKeys(inputId, myAddedKeyProcessor, myUpdatedKeyProcessor, myRemovedKeyProcessor);
+        updateData.updateForwardIndex();
+      }
+      catch (ProcessCanceledException e) {
+        throw e;
+      }
+      catch (Throwable e) { // e.g. IOException, AssertionError
+        throw new StorageException(e);
+      }
+      finally {
+        ValueContainerImpl.ourDebugIndexInfo.set(null);
+      }
+    }
+    finally {
+      getWriteLock().unlock();
+    }
+  }
+
+  public static <Key, Value> void checkValuesHaveProperEqualsAndHashCode(@NotNull Map<Key, Value> data,
+                                                                         @NotNull ID<Key, Value> indexId,
+                                                                         @NotNull DataExternalizer<Value> valueExternalizer) {
+    if (DebugAssertions.DEBUG) {
+      for (Map.Entry<Key, Value> e : data.entrySet()) {
+        final Value value = e.getValue();
+        if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) {
+          LOG.error("Index " + indexId.toString() + " violates equals / hashCode contract for Value parameter");
+        }
+
+        try {
+          final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
+          DataOutputStream outputStream = new DataOutputStream(out);
+          valueExternalizer.save(outputStream, value);
+          outputStream.close();
+          final Value deserializedValue =
+            valueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size())));
+
+          if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) {
+            LOG.error("Index " + indexId.toString() + " deserialization violates equals / hashCode contract for Value parameter");
+          }
+        }
+        catch (IOException ex) {
+          LOG.error(ex);
+        }
+      }
+    }
+  }
+}
+

platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.openapi.util.ThrowableComputable;
+import com.intellij.util.ThrowableRunnable;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.StorageException;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class SimpleUpdateData<Key, Value> extends UpdateData<Key,Value> {
+  public SimpleUpdateData(@NotNull Map<Key, Value> newData,
+                          @NotNull ThrowableComputable<ForwardIndex.InputKeyIterator<Key, Value>, IOException> currentData,
+                          @NotNull ID<Key, Value> indexId,
+                          ThrowableRunnable<IOException> forwardIndexUpdate) {
+    super(newData, currentData, indexId, forwardIndexUpdate);
+  }
+
+  @Override
+  public void iterateKeys(int inputId,
+                          KeyValueUpdateProcessor<Key, Value> addProcessor,
+                          KeyValueUpdateProcessor<Key, Value> updateProcessor,
+                          RemovedKeyProcessor<Key> removeProcessor) throws StorageException {
+    final ForwardIndex.InputKeyIterator<Key, Value> currentData;
+    try {
+      currentData = myCurrentData.compute();
+    }
+    catch (IOException e) {
+      throw new StorageException(e);
+    }
+    iterateKeys(inputId, addProcessor, removeProcessor, currentData);
+  }
+
+  protected void iterateKeys(int inputId,
+                             KeyValueUpdateProcessor<Key, Value> addProcessor,
+                             RemovedKeyProcessor<Key> removeProcessor, ForwardIndex.InputKeyIterator<Key, Value> currentData)
+    throws StorageException {
+    while (currentData.hasNext()) {
+      removeProcessor.process(currentData.next(), inputId);
+    }
+    for (Map.Entry<Key, Value> entry : myNewData.entrySet()) {
+      addProcessor.process(entry.getKey(), entry.getValue(), inputId);
+    }
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/UpdatableValueContainer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,8 +14,9 @@
  * limitations under the License.
  */
 
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
+import com.intellij.util.indexing.ValueContainer;
 import com.intellij.util.io.DataExternalizer;
 
 import java.io.DataOutput;
@@ -25,7 +26,7 @@ import java.io.IOException;
  * @author Eugene Zhuravlev
  *         Date: Feb 27, 2008
  */
-public abstract class UpdatableValueContainer<T> extends ValueContainer<T>{
+public abstract class UpdatableValueContainer<T> extends ValueContainer<T> {
 
   public abstract void addValue(int inputId, T value);
 

platform/util/src/com/intellij/util/indexing/impl/UpdateData.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.util.indexing.impl;
+
+import com.intellij.openapi.util.ThrowableComputable;
+import com.intellij.util.ThrowableRunnable;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.StorageException;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.util.Map;
+
+@ApiStatus.Experimental
+public abstract class UpdateData<Key, Value> {
+  protected final Map<Key, Value> myNewData;
+  protected final ThrowableComputable<ForwardIndex.InputKeyIterator<Key, Value>, IOException> myCurrentData;
+  private final ID<Key, Value> myIndexId;
+  private final ThrowableRunnable<IOException> myForwardIndexUpdate;
+
+  protected UpdateData(@NotNull Map<Key, Value> newData,
+                       @NotNull ThrowableComputable<ForwardIndex.InputKeyIterator<Key, Value>, IOException> currentData,
+                       @NotNull ID<Key, Value> indexId,
+                       @Nullable ThrowableRunnable<IOException> forwardIndexUpdate) {
+    myNewData = newData;
+    myCurrentData = currentData;
+    myIndexId = indexId;
+    myForwardIndexUpdate = forwardIndexUpdate;
+  }
+
+  public abstract void iterateKeys(final int inputId,
+                                   final KeyValueUpdateProcessor<Key, Value> addProcessor,
+                                   final KeyValueUpdateProcessor<Key, Value> updateProcessor,
+                                   final RemovedKeyProcessor<Key> removeProcessor) throws StorageException;
+
+  public Map<Key, Value> getNewData() {
+    return myNewData;
+  }
+
+  public interface KeyValueUpdateProcessor<Key, Value> {
+    void process(Key key, Value value, int inputId) throws StorageException;
+  }
+
+  public interface RemovedKeyProcessor<Key> {
+    void process(Key key, int inputId) throws StorageException;
+  }
+
+  public ID<Key, Value> getIndexId() {
+    return myIndexId;
+  }
+
+  public void updateForwardIndex() throws IOException {
+    if (myForwardIndexUpdate != null) {
+      myForwardIndexUpdate.run();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return myIndexId + "," + getClass().getName();
+  }
+}

platform/util/src/com/intellij/util/indexing/impl/ValueContainerImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.util.SmartList;
 import com.intellij.util.containers.EmptyIterator;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.ValueContainer;
 import com.intellij.util.indexing.containers.ChangeBufferingList;
 import com.intellij.util.indexing.containers.IdSet;
 import com.intellij.util.indexing.containers.IntIdsIterator;
@@ -40,7 +42,7 @@ import java.util.List;
  *         Date: Dec 20, 2007
  */
 class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implements Cloneable{
-  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.ValueContainerImpl");
+  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.ValueContainerImpl");
   private final static Object myNullValue = new Object();
 
   // there is no volatile as we modify under write lock and read under read lock
@@ -78,7 +80,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
     return myInputIdMapping != null ? myInputIdMapping instanceof THashMap ? ((THashMap)myInputIdMapping).size(): 1 : 0;
   }
 
-  static final ThreadLocal<ID> ourDebugIndexInfo = new ThreadLocal<>();
+  static final ThreadLocal<ID> ourDebugIndexInfo = new ThreadLocal<ID>();
 
   @Override
   public void removeAssociatedValue(int inputId) {
@@ -90,8 +92,8 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
 
       if (valueIterator.getValueAssociationPredicate().contains(inputId)) {
         if (fileSetObjects == null) {
-          fileSetObjects = new SmartList<>();
-          valueObjects = new SmartList<>();
+          fileSetObjects = new SmartList<Object>();
+          valueObjects = new SmartList<Value>();
         }
         else if (DebugAssertions.DEBUG) {
           LOG.error("Expected only one value per-inputId for " + ourDebugIndexInfo.get(), String.valueOf(fileSetObjects.get(0)), String.valueOf(value));
@@ -151,7 +153,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
 
           @NotNull
           @Override
-          public IntIterator getInputIdsIterator() {
+          public ValueContainer.IntIterator getInputIdsIterator() {
             return getIntIteratorOutOfFileSetObject(getFileSetObject());
           }
 
@@ -212,7 +214,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
 
           @NotNull
           @Override
-          public IntIterator getInputIdsIterator() {
+          public ValueContainer.IntIterator getInputIdsIterator() {
             return getIntIteratorOutOfFileSetObject(getFileSetObject());
           }
 
@@ -238,7 +240,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
 
     @NotNull
     @Override
-    public IntIterator getInputIdsIterator() {
+    public ValueContainer.IntIterator getInputIdsIterator() {
       throw new IllegalStateException();
     }
 
@@ -272,7 +274,8 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
     return ((ChangeBufferingList)input).intPredicate();
   }
 
-  private static @NotNull IntIterator getIntIteratorOutOfFileSetObject(@Nullable Object input) {
+  private static @NotNull
+  ValueContainer.IntIterator getIntIteratorOutOfFileSetObject(@Nullable Object input) {
     if (input == null) return EMPTY_ITERATOR;
     if (input instanceof Integer){
       return new SingleValueIterator(((Integer)input).intValue());
@@ -312,7 +315,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
     }
   }
 
-  private static final IntIterator EMPTY_ITERATOR = new IntIdsIterator() {
+  private static final ValueContainer.IntIterator EMPTY_ITERATOR = new IntIdsIterator() {
     @Override
     public boolean hasNext() {
       return false;
@@ -341,11 +344,11 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
 
   @NotNull
   public ValueContainerImpl<Value> copy() {
-    ValueContainerImpl<Value> container = new ValueContainerImpl<>();
+    ValueContainerImpl<Value> container = new ValueContainerImpl<Value>();
 
     if (myInputIdMapping instanceof THashMap) {
       final THashMap<Value, Object> mapping = (THashMap<Value, Object>)myInputIdMapping;
-      final THashMap<Value, Object> newMapping = new THashMap<>(mapping.size());
+      final THashMap<Value, Object> newMapping = new THashMap<Value, Object>(mapping.size());
       container.myInputIdMapping = newMapping;
 
       mapping.forEachEntry(new TObjectObjectProcedure<Value, Object>() {
@@ -460,7 +463,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
         final int inputId = -valueCount;
 
         if (mapping == null && size() > NUMBER_OF_VALUES_THRESHOLD) { // avoid O(NumberOfValues)
-          mapping = new FileId2ValueMapping<>(this);
+          mapping = new FileId2ValueMapping<Value>(this);
         }
 
         boolean doCompact;

platform/util/src/com/intellij/util/indexing/impl/ValueContainerMap.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2016 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.intellij.util.indexing;
+package com.intellij.util.indexing.impl;
 
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.KeyDescriptor;
@@ -35,7 +35,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, UpdatableValu
                     @NotNull DataExternalizer<Value> valueExternalizer,
                     boolean keyIsUniqueForIndexedFile
                     ) throws IOException {
-    super(file, keyKeyDescriptor, new ValueContainerExternalizer<>(valueExternalizer));
+    super(file, keyKeyDescriptor, new ValueContainerExternalizer<Value>(valueExternalizer));
     myValueExternalizer = valueExternalizer;
     myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
   }
@@ -48,7 +48,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, UpdatableValu
   @Override
   protected void doPut(Key key, UpdatableValueContainer<Value> container) throws IOException {
     synchronized (myEnumerator) {
-      ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
+      final ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
 
       // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files
       // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular
@@ -83,7 +83,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, UpdatableValu
     @NotNull
     @Override
     public UpdatableValueContainer<T> read(@NotNull final DataInput in) throws IOException {
-      final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<>();
+      final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
 
       valueContainer.readFrom((DataInputStream)in, myValueExternalizer);
       return valueContainer;

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java

@@ -16,23 +16,27 @@
 package com.intellij.vcs.log.data.index;
 
 import com.intellij.openapi.Disposable;
+import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.util.Consumer;
 import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.impl.EmptyInputKeyIterator;
+import com.intellij.util.indexing.impl.ForwardIndex;
+import com.intellij.util.indexing.impl.MapIndexStorage;
+import com.intellij.util.indexing.impl.MapReduceIndex;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.EnumeratorIntegerDescriptor;
 import com.intellij.util.io.KeyDescriptor;
-import com.intellij.util.io.PersistentHashMap;
 import com.intellij.vcs.log.VcsFullCommitDetails;
 import com.intellij.vcs.log.impl.FatalErrorHandler;
 import com.intellij.vcs.log.util.PersistentUtil;
 import gnu.trove.TIntHashSet;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.ObjIntConsumer;
 
@@ -78,35 +82,29 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
   }
 
   @NotNull
-  public ValueContainer.IntIterator getCommitsWithAllKeys(@NotNull Collection<Integer> keys) throws StorageException {
-    return FileBasedIndexImpl.collectInputIdsContainingAllKeys(myMapReduceIndex, keys);
+  public TIntHashSet getCommitsWithAllKeys(@NotNull Collection<Integer> keys) throws StorageException {
+    return InvertedIndexUtil.collectInputIdsContainingAllKeys(myMapReduceIndex, keys, (k) -> {
+      ProgressManager.checkCanceled();
+      return true;
+    }, null, null);
   }
 
   private void iterateCommitIds(int key, @NotNull Consumer<Integer> consumer) throws StorageException {
     ValueContainer<T> data = myMapReduceIndex.getData(key);
-
-    ValueContainer.ValueIterator<T> valueIt = data.getValueIterator();
-    while (valueIt.hasNext()) {
-      valueIt.next();
-      ValueContainer.IntIterator inputIt = valueIt.getInputIdsIterator();
-      while (inputIt.hasNext()) {
-        consumer.consume(inputIt.next());
+    data.forEach(new ValueContainer.ContainerAction<T>() {
+      @Override
+      public boolean perform(int id, T value) {
+        consumer.consume(id);
+        return true;
       }
-    }
+    });
   }
 
   protected void iterateCommitIdsAndValues(int key, @NotNull ObjIntConsumer<T> consumer) throws StorageException {
-    ValueContainer<T> data = myMapReduceIndex.getData(key);
-
-    ValueContainer.ValueIterator<T> valueIt = data.getValueIterator();
-    while (valueIt.hasNext()) {
-      T nextValue = valueIt.next();
-      ValueContainer.IntIterator inputIt = valueIt.getInputIdsIterator();
-      while (inputIt.hasNext()) {
-        int next = inputIt.next();
-        consumer.accept(nextValue, next);
-      }
-    }
+    myMapReduceIndex.getData(key).forEach((id, value) -> {
+      consumer.accept(value, id);
+      return true;
+    });
   }
 
   public void update(int commitId, @NotNull VcsFullCommitDetails details) throws IOException {
@@ -128,24 +126,55 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
   }
 
   private class MyMapReduceIndex extends MapReduceIndex<Integer, T, VcsFullCommitDetails> {
-
     public MyMapReduceIndex(@NotNull DataIndexer<Integer, T, VcsFullCommitDetails> indexer,
                             @NotNull DataExternalizer<T> externalizer,
                             int version) throws IOException {
       super(new MyIndexExtension(indexer, externalizer, version),
-            new MapIndexStorage<>(getStorageFile(myName, myLogId),
-                                  EnumeratorIntegerDescriptor.INSTANCE,
-                                  externalizer, 5000));
+            new MapIndexStorage<Integer, T>(getStorageFile(myName, myLogId),
+                                            EnumeratorIntegerDescriptor.INSTANCE,
+                                            externalizer, 5000, false) {
+              @Override
+              protected void checkCanceled() {
+                ProgressManager.checkCanceled();
+              }
+            },
+            new ForwardIndex<Integer, T>() {
+              @NotNull
+              @Override
+              public InputKeyIterator<Integer, T> getInputKeys(int inputId) {
+                return EmptyInputKeyIterator.getInstance();
+              }
+
+              @Override
+              public void putInputData(int inputId, @NotNull Map<Integer, T> data) throws IOException {
+
+              }
+
+              @Override
+              public void flush() {
+
+              }
+
+              @Override
+              public void clear() throws IOException {
+
+              }
+
+              @Override
+              public void close() throws IOException {
+
+              }
+            });
     }
 
     @Override
-    protected PersistentHashMap<Integer, Collection<Integer>> createInputsIndex() throws IOException {
-      return null;
+    public void checkCanceled() {
+      ProgressManager.checkCanceled();
     }
 
     @Override
-    protected void requestRebuild(@Nullable Exception ex) {
-      myFatalErrorHandler.consume(this, ex != null ? ex : new Exception("Index rebuild requested"));
+    public void requestRebuild(@NotNull Exception ex) {
+      myFatalErrorHandler.consume(this, ex);
     }
   }
 

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java

@@ -19,11 +19,11 @@ import com.intellij.openapi.Disposable;
 import com.intellij.openapi.util.text.TrigramBuilder;
 import com.intellij.util.indexing.DataIndexer;
 import com.intellij.util.indexing.StorageException;
-import com.intellij.util.indexing.ValueContainer;
 import com.intellij.util.io.VoidDataExternalizer;
 import com.intellij.vcs.log.VcsFullCommitDetails;
 import com.intellij.vcs.log.impl.FatalErrorHandler;
 import gnu.trove.THashMap;
+import gnu.trove.TIntHashSet;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -43,7 +43,7 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex<Void> {
   }
 
   @Nullable
-  public ValueContainer.IntIterator getCommitsForSubstring(@NotNull String string) throws StorageException {
+  public TIntHashSet getCommitsForSubstring(@NotNull String string) throws StorageException {
     MyTrigramProcessor trigramProcessor = new MyTrigramProcessor();
     TrigramBuilder.processTrigrams(string, trigramProcessor);
 

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java

@@ -33,7 +33,6 @@ import com.intellij.util.ThrowableRunnable;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.containers.EmptyIntHashSet;
 import com.intellij.util.indexing.StorageException;
-import com.intellij.util.indexing.ValueContainer;
 import com.intellij.util.io.*;
 import com.intellij.vcs.log.*;
 import com.intellij.vcs.log.data.*;
@@ -270,11 +269,10 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
           return filter(myIndexStorage.messages, message -> pattern.matcher(message).find());
         }
         else {
-          ValueContainer.IntIterator commitsForSearch = myIndexStorage.trigrams.getCommitsForSubstring(text);
+          TIntHashSet commitsForSearch = myIndexStorage.trigrams.getCommitsForSubstring(text);
           if (commitsForSearch != null) {
             TIntHashSet result = new TIntHashSet();
-            while (commitsForSearch.hasNext()) {
-              int commit = commitsForSearch.next();
+            commitsForSearch.forEach(commit -> {
               try {
                 String value = myIndexStorage.messages.get(commit);
                 if (value != null) {
@@ -285,9 +283,10 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable {
               }
               catch (IOException e) {
                 myFatalErrorsConsumer.consume(this, e);
-                break;
+                return false;
               }
-            }
+              return true;
+            });
             return result;
           }
         }