allow to load multiple shared indexes

GitOrigin-RevId: 9b108cbe7c616a5f7885fb48f468b574a48626a2

java/java-impl/src/com/intellij/internal/DumpIndexAction.java

@@ -2,110 +2,292 @@
 package com.intellij.internal;
 
 import com.intellij.codeInspection.bytecodeAnalysis.BytecodeAnalysisIndex;
+import com.intellij.concurrency.JobLauncher;
 import com.intellij.find.ngrams.TrigramIndex;
 import com.intellij.openapi.actionSystem.AnAction;
 import com.intellij.openapi.actionSystem.AnActionEvent;
+import com.intellij.openapi.application.ReadAction;
+import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.fileChooser.FileChooser;
 import com.intellij.openapi.fileChooser.FileChooserDescriptor;
 import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory;
+import com.intellij.openapi.module.Module;
 import com.intellij.openapi.module.ModuleManager;
+import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.openapi.progress.Task;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.roots.*;
 import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.VirtualFileUrlChangeAdapter;
+import com.intellij.psi.impl.JavaSimplePropertyIndex;
 import com.intellij.psi.impl.cache.impl.id.IdIndex;
+import com.intellij.psi.impl.cache.impl.todo.TodoIndex;
+import com.intellij.psi.impl.java.JavaBinaryPlusExpressionIndex;
+import com.intellij.psi.impl.java.JavaFunctionalExpressionIndex;
 import com.intellij.psi.impl.java.stubs.index.JavaAutoModuleNameIndex;
-import com.intellij.psi.stubs.StubIndexKey;
+import com.intellij.psi.impl.search.JavaNullMethodArgumentIndex;
+import com.intellij.psi.stubs.StubUpdatingIndex;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.FileBasedIndexExtension;
 import com.intellij.util.indexing.ID;
 import com.intellij.util.indexing.IndexableSetContributor;
 import com.intellij.util.indexing.hash.HashBasedIndexGenerator;
-import one.util.streamex.StreamEx;
+import com.intellij.util.io.zip.JBZipEntry;
+import com.intellij.util.io.zip.JBZipFile;
+import gnu.trove.THashSet;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.index.stubs.StubHashBasedIndexGenerator;
 
 import java.io.File;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
 
 public class DumpIndexAction extends AnAction {
+  private static final Logger LOG = Logger.getInstance(DumpIndexAction.class);
 
   @Override
   public void actionPerformed(@NotNull AnActionEvent event) {
     Project project = event.getProject();
     if (project == null) return;
-    Stream<VirtualFile> libRoots = Arrays
-      .stream(ModuleManager.getInstance(project).getModules())
-      .flatMap(m -> Arrays.stream(ModuleRootManager.getInstance(m).getOrderEntries()))
-      .filter(e -> e instanceof LibraryOrSdkOrderEntry)
-      .flatMap(e -> Stream.concat(Arrays.stream(e.getFiles(OrderRootType.CLASSES)), Arrays.stream(e.getFiles(OrderRootType.SOURCES))));
+    Collection<IndexChunk> projectChunks = Arrays
+            .stream(ModuleManager.getInstance(project).getModules())
+            .flatMap(m -> IndexChunk.generate(m))
+            .collect(Collectors.toMap(ch -> ch.getName(), ch -> ch, IndexChunk::mergeUnsafe))
+            .values();
 
-    Stream<VirtualFile> additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(),
-                                                                                                                            IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream()));
-    Set<VirtualFile> roots = Stream.concat(libRoots, additionalRoots).collect(Collectors.toSet());
+    Set<VirtualFile>
+            additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(),
+            IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())).collect(
+            Collectors.toSet());
+
+    Set<VirtualFile> synthRoots = new THashSet<>();
+    for (AdditionalLibraryRootsProvider provider : AdditionalLibraryRootsProvider.EP_NAME.getExtensionList()) {
+      for (SyntheticLibrary library : provider.getAdditionalProjectLibraries(project)) {
+        for (VirtualFile root : library.getAllRoots()) {
+          // do not try to visit under-content-roots because the first task took care of that already
+          if (!ProjectFileIndex.getInstance(project).isInContent(root)) {
+            synthRoots.add(root);
+          }
+        }
+      }
+    }
+
+    List<IndexChunk> chunks = Stream.concat(projectChunks.stream(),
+            Stream.of(new IndexChunk(additionalRoots, "ADDITIONAL"),
+                    new IndexChunk(synthRoots, "SYNTH"))).collect(Collectors.toList());
+
+    //IndexChunk chunk = chunks.stream().reduce((c1, c2) -> IndexChunk.mergeUnsafe(c1, c2)).get();
 
     FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor();
     descriptor.withTitle("Select Index Dump Directory");
     VirtualFile file = FileChooser.chooseFile(descriptor, project, null);
-    if (file == null) return;
-    File out = VfsUtilCore.virtualToIoFile(file);
-    FileUtil.delete(out);
-    exportIndices(roots, out);
+    if (file != null) {
+      ProgressManager.getInstance().run(new Task.Modal(project, "Exporting Indexes..." , true) {
+        @Override
+        public void run(@NotNull ProgressIndicator indicator) {
+          File out = VfsUtilCore.virtualToIoFile(file);
+          FileUtil.delete(out);
+          exportIndices(chunks, out, indicator, project);
+        }
+      });
+    }
   }
 
-  public static void exportIndices(@NotNull Set<VirtualFile> roots, @NotNull File out) {
-    StubHashBasedIndexGenerator generator = new StubHashBasedIndexGenerator(out);
-    generator.generate(roots);
+  public static void exportIndices(@NotNull List<IndexChunk> chunks,
+                                   @NotNull File out,
+                                   @NotNull ProgressIndicator indicator,
+                                   @NotNull Project project) {
+    indicator.setIndeterminate(false);
+    AtomicInteger idx = new AtomicInteger();
+    if (!JobLauncher.getInstance().invokeConcurrentlyUnderProgress(chunks, indicator, chunk -> {
+      indicator.setText("Indexing '" + chunk.getName() + "' chunk");
+      File chunkOut = new File(out, chunk.getName());
+      ReadAction.run(() -> {
+        Stream<HashBasedIndexGenerator<?, ?>> fbIndexes = getExportableIndices(true).map(ex -> new HashBasedIndexGenerator(ex, chunkOut));
+        Stream<HashBasedIndexGenerator<?, ?>> stubIndex = Stream.of(new StubHashBasedIndexGenerator(chunkOut));
+        List<HashBasedIndexGenerator<?, ?>> indexes = Stream.concat(fbIndexes, stubIndex).collect(Collectors.toList());
+        HashBasedIndexGenerator.generate(chunk.getRoots(),indexes, project, chunkOut);
+      });
+      indicator.setFraction(((double) idx.incrementAndGet()) / chunks.size());
+      return true;
+    })) {
+      throw new AssertionError();
+    }
 
-    getExportableIndices().forEach(ex -> {
-      HashBasedIndexGenerator indexGenerator = new HashBasedIndexGenerator(ex, out);
-      indexGenerator.generate(roots);
-    });
+    indicator.setIndeterminate(true);
+    indicator.setText("Zipping index pack");
+
+    File zipFile = new File(out.getAbsolutePath() + ".zip");
+    FileUtil.delete(zipFile);
+    try (JBZipFile file = new JBZipFile(zipFile)) {
+      Path outPath = out.toPath();
+      Files.walk(outPath).forEach(p -> {
+        if (Files.isDirectory(p)) return;
+        String relativePath = outPath.relativize(p).toString();
+        try {
+          JBZipEntry entry = file.getOrCreateEntry(relativePath);
+          entry.setMethod(ZipEntry.STORED);
+          entry.setDataFromFile(p.toFile());
+        }
+        catch (IOException e) {
+          LOG.error(e);
+        }
+      });
+    }
+    catch (IOException e) {
+      LOG.error(e);
+    }
   }
 
   @NotNull
-  private static Stream<FileBasedIndexExtension> getExportableIndices() {
+  private static Stream<FileBasedIndexExtension> getExportableIndices(boolean all) {
+    if (all) {
+      return FileBasedIndexExtension
+              .EXTENSION_POINT_NAME
+              .extensions()
+              .filter(ex -> ex.dependsOnFileContent())
+              .filter(ex -> !(ex instanceof StubUpdatingIndex));
+    }
+
     //kt
     Stream<FileBasedIndexExtension> ktIndices =
-      FileBasedIndexExtension
-        .EXTENSION_POINT_NAME
-        .extensions()
-        .filter(id -> id.getName().getName().contains("kotlin"));
+            FileBasedIndexExtension
+                    .EXTENSION_POINT_NAME
+                    .extensions()
+                    .filter(id -> id.getName().getName().contains("kotlin"));
 
     Set<ID<Object, Object>> xmlIndexIds = ContainerUtil.set(ID.findByName("XmlTagNames"),
-                                                            ID.findByName("XmlNamespaces"),
-                                                            ID.findByName("SchemaTypeInheritance"),
-                                                            ID.findByName("DomFileIndex"),
-                                                            ID.findByName("xmlProperties"));
+            ID.findByName("XmlNamespaces"),
+            ID.findByName("SchemaTypeInheritance"),
+            ID.findByName("DomFileIndex"),
+            ID.findByName("xmlProperties"));
     //xml
     Stream<FileBasedIndexExtension> xmlIndices =
-      FileBasedIndexExtension
-        .EXTENSION_POINT_NAME
-        .extensions()
-        .filter(id -> xmlIndexIds.contains(id.getName()));
+            FileBasedIndexExtension
+                    .EXTENSION_POINT_NAME
+                    .extensions()
+                    .filter(id -> xmlIndexIds.contains(id.getName()));
 
     //base
     Stream<FileBasedIndexExtension> coreIndices =
-      FileBasedIndexExtension
-        .EXTENSION_POINT_NAME
-        .extensions()
-        .filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) ||
-                      ex.getName().equals(IdIndex.NAME));
+            FileBasedIndexExtension
+                    .EXTENSION_POINT_NAME
+                    .extensions()
+                    .filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) ||
+                            ex.getName().equals(TodoIndex.NAME) ||
+                            ex.getName().equals(IdIndex.NAME) ||
+                            ex.getName().getName().equals("HashFragmentIndex"));
 
     //java
     Stream<FileBasedIndexExtension> javaIndices =
-      FileBasedIndexExtension
-        .EXTENSION_POINT_NAME
-        .extensions()
-        .filter(ex -> ex instanceof BytecodeAnalysisIndex || ex instanceof JavaAutoModuleNameIndex);
+            FileBasedIndexExtension
+                    .EXTENSION_POINT_NAME
+                    .extensions()
+                    .filter(ex -> ex instanceof BytecodeAnalysisIndex ||
+                            ex instanceof JavaAutoModuleNameIndex ||
+                            ex instanceof JavaFunctionalExpressionIndex ||
+                            ex instanceof JavaSimplePropertyIndex ||
+                            ex instanceof JavaNullMethodArgumentIndex ||
+                            ex instanceof JavaBinaryPlusExpressionIndex);
 
     return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices);
   }
+
+  private static final class IndexChunk {
+    private final Set<VirtualFile> myRoots;
+    private final String myName;
+
+    IndexChunk(Set<VirtualFile> roots, String name) {
+      myRoots = roots;
+      myName = name;
+    }
+
+    private String getName() {
+      return myName;
+    }
+
+    private Set<VirtualFile> getRoots() {
+      return myRoots;
+    }
+
+    static IndexChunk mergeUnsafe(IndexChunk ch1, IndexChunk ch2) {
+      ch1.getRoots().addAll(ch2.getRoots());
+      return ch1;
+    }
+
+    static Stream<IndexChunk> generate(Module module) {
+      Stream<IndexChunk> libChunks = Arrays.stream(ModuleRootManager.getInstance(module).getOrderEntries())
+              .map(orderEntry -> {
+                if (orderEntry instanceof LibraryOrSdkOrderEntry) {
+                  VirtualFile[] sources = orderEntry.getFiles(OrderRootType.SOURCES);
+                  VirtualFile[] classes = orderEntry.getFiles(OrderRootType.CLASSES);
+                  String name = null;
+                  if (orderEntry instanceof JdkOrderEntry) {
+                    name = ((JdkOrderEntry)orderEntry).getJdkName();
+                  }
+                  else if (orderEntry instanceof LibraryOrderEntry) {
+                    name = ((LibraryOrderEntry)orderEntry).getLibraryName();
+                  }
+                  if (name == null) {
+                    name = "unknown";
+                  }
+                  return new IndexChunk(ContainerUtil.union(Arrays.asList(sources), Arrays.asList(classes)), reducePath(splitByDots(name)));
+                }
+                return null;
+              })
+              .filter(Objects::nonNull);
+
+      Set<VirtualFile> roots =
+              ContainerUtil.union(ContainerUtil.newTroveSet(ModuleRootManager.getInstance(module).getContentRoots()),
+              ContainerUtil.newTroveSet(ModuleRootManager.getInstance(module).getSourceRoots()));
+      Stream<IndexChunk> srcChunks = Stream.of(new IndexChunk(roots, getChunkName(module)));
+
+      return Stream.concat(libChunks, srcChunks);
+    }
+
+    private static String getChunkName(Module module) {
+      ModuleManager moduleManager = ModuleManager.getInstance(module.getProject());
+      String[] path;
+      if (moduleManager.hasModuleGroups()) {
+        path = moduleManager.getModuleGroupPath(module);
+        assert path != null;
+      } else {
+        path = splitByDots(module.getName());
+      }
+      return reducePath(path);
+    }
+
+    @NotNull
+    private static String reducePath(String[] path) {
+      String[] reducedPath = Arrays.copyOfRange(path, 0, Math.min(1, path.length));
+      return StringUtil.join(reducedPath, ".");
+    }
+
+    private static String[] splitByDots(String name) {
+      return name.split("[-|:.]");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+      IndexChunk chunk = (IndexChunk)o;
+      return Objects.equals(myRoots, chunk.myRoots) &&
+              Objects.equals(myName, chunk.myName);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(myRoots, myName);
+    }
+  }
 }

platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java

@@ -45,11 +45,16 @@ public class SerializedStubTree {
   private Map<StubIndexKey, Map<Object, StubIdList>> myIndexedStubs;
 
   private volatile SerializationManagerEx mySerializationManager;
+  private volatile StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
 
-  public void setSerializationManager(SerializationManagerEx serializationManager) {
+  public void setSerializationManager(@NotNull SerializationManagerEx serializationManager) {
     mySerializationManager = serializationManager;
   }
 
+  public void setStubIndexesExternalizer(@NotNull StubForwardIndexExternalizer<?> stubIndexesExternalizer) {
+    myStubIndexesExternalizer = stubIndexesExternalizer;
+  }
+
   public SerializedStubTree(@NotNull byte[] treeBytes, int treeByteLength, @Nullable Stub stubElement,
                             @NotNull byte[] indexedStubBytes, int indexedStubByteLength, @Nullable Map<StubIndexKey, Map<Object, StubIdList>> indexedStubs) {
     myTreeBytes = treeBytes;
@@ -74,6 +79,7 @@ public class SerializedStubTree {
     forwardIndexExternalizer.save(new DataOutputStream(indexBytes), myIndexedStubs);
     myIndexedStubBytes = indexBytes.getInternalBuffer();
     myIndexedStubByteLength = indexBytes.size();
+    myStubIndexesExternalizer = forwardIndexExternalizer;
   }
 
   @NotNull
@@ -94,7 +100,7 @@ public class SerializedStubTree {
     else {
       BufferExposingByteArrayOutputStream reSerializedStubIndices = new BufferExposingByteArrayOutputStream();
       if (myIndexedStubs == null) {
-        restoreIndexedStubs(currentForwardIndexSerializer);
+        restoreIndexedStubs();
       }
       assert myIndexedStubs != null;
       newForwardIndexSerializer.save(new DataOutputStream(reSerializedStubIndices), myIndexedStubs);
@@ -102,18 +108,25 @@ public class SerializedStubTree {
       reSerializedIndexByteLength = reSerializedStubIndices.size();
     }
 
-    return new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null,
-                                  reSerializedIndexBytes, reSerializedIndexByteLength, myIndexedStubs);
+    SerializedStubTree tree = new SerializedStubTree(
+            outStub.getInternalBuffer(),
+            outStub.size(),
+            null,
+            reSerializedIndexBytes,
+            reSerializedIndexByteLength,
+            myIndexedStubs);
+    tree.setStubIndexesExternalizer(myStubIndexesExternalizer);
+    return tree;
   }
 
-  void restoreIndexedStubs(@NotNull StubForwardIndexExternalizer<?> dataExternalizer) throws IOException {
+  void restoreIndexedStubs() throws IOException {
     if (myIndexedStubs == null) {
-      myIndexedStubs = dataExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)));
+      myIndexedStubs = myStubIndexesExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)));
     }
   }
 
-  <K> StubIdList restoreIndexedStubs(@NotNull StubForwardIndexExternalizer<?> dataExternalizer, @NotNull StubIndexKey<K, ?> indexKey, @NotNull K key) throws IOException {
-    Map<StubIndexKey, Map<Object, StubIdList>> incompleteMap = dataExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key);
+  <K> StubIdList restoreIndexedStubs(@NotNull StubIndexKey<K, ?> indexKey, @NotNull K key) throws IOException {
+    Map<StubIndexKey, Map<Object, StubIdList>> incompleteMap = myStubIndexesExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key);
     Map<Object, StubIdList> map = incompleteMap.get(indexKey);
     return map == null ? null : map.get(key);
   }
@@ -125,7 +138,7 @@ public class SerializedStubTree {
 
   @TestOnly
   public Map<StubIndexKey, Map<Object, StubIdList>> readStubIndicesValueMap() throws IOException {
-    restoreIndexedStubs(IDE_USED_EXTERNALIZER);
+    restoreIndexedStubs();
     return myIndexedStubs;
   }
 

platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java

@@ -15,14 +15,16 @@ import java.io.IOException;
 public class SerializedStubTreeDataExternalizer implements DataExternalizer<SerializedStubTree> {
   private final boolean myIncludeInputs;
   private final SerializationManagerEx mySerializationManager;
+  private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
 
   public SerializedStubTreeDataExternalizer() {
-    this(true, null);
+    this(true, null, SerializedStubTree.IDE_USED_EXTERNALIZER);
   }
 
-  public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager) {
+  public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager, StubForwardIndexExternalizer<?> externalizer) {
     myIncludeInputs = inputs;
     mySerializationManager = manager;
+    myStubIndexesExternalizer = externalizer;
   }
 
   @Override
@@ -44,6 +46,7 @@ public class SerializedStubTreeDataExternalizer implements DataExternalizer<Seri
   @NotNull
   @Override
   public final SerializedStubTree read(@NotNull final DataInput in) throws IOException {
+    SerializedStubTree tree;
     if (PersistentHashMapValueStorage.COMPRESSION_ENABLED) {
       int serializedStubsLength = DataInputOutputUtil.readINT(in);
       byte[] bytes = new byte[serializedStubsLength];
@@ -58,14 +61,15 @@ public class SerializedStubTreeDataExternalizer implements DataExternalizer<Seri
         indexedStubByteLength = 0;
         indexedStubBytes = ArrayUtil.EMPTY_BYTE_ARRAY;
       }
-      SerializedStubTree tree = new SerializedStubTree(bytes, bytes.length, null, indexedStubBytes, indexedStubByteLength, null);
-      if (mySerializationManager != null) tree.setSerializationManager(mySerializationManager);
-      return tree;
+      tree = new SerializedStubTree(bytes, bytes.length, null, indexedStubBytes, indexedStubByteLength, null);
     }
     else {
       byte[] treeBytes = CompressionUtil.readCompressed(in);
       byte[] indexedStubBytes = myIncludeInputs ? CompressionUtil.readCompressed(in) : ArrayUtil.EMPTY_BYTE_ARRAY;
-      return new SerializedStubTree(treeBytes, treeBytes.length, null, indexedStubBytes, indexedStubBytes.length, null);
+      tree = new SerializedStubTree(treeBytes, treeBytes.length, null, indexedStubBytes, indexedStubBytes.length, null);
     }
+    if (mySerializationManager != null) tree.setSerializationManager(mySerializationManager);
+    tree.setStubIndexesExternalizer(myStubIndexesExternalizer);
+    return tree;
   }
 }

platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java

@@ -2,6 +2,7 @@
 package com.intellij.index;
 
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.ShutDownTracker;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.psi.stubs.StubUpdatingIndex;
 import com.intellij.psi.stubs.provided.StubProvidedIndexExtension;
@@ -9,44 +10,73 @@ import com.intellij.util.indexing.FileBasedIndexExtension;
 import com.intellij.util.indexing.ID;
 import com.intellij.util.indexing.provided.ProvidedIndexExtension;
 import com.intellij.util.indexing.provided.ProvidedIndexExtensionLocator;
+import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem;
+import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.KeyDescriptor;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.net.URI;
-import java.net.URISyntaxException;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.stream.Stream;
 
 public class BasicProvidedExtensionLocator implements ProvidedIndexExtensionLocator {
-  private static final String PREBUILT_INDEX_PATH_PROP = "prebuilt.hash.index.dir";
+  private static final String PREBUILT_INDEX_ZIP_PROP = "prebuilt.hash.index.zip";
   private static final Logger LOG = Logger.getInstance(BasicProvidedExtensionLocator.class);
 
-  @Nullable
-  @Override
-  public <K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
-    Path root = getPrebuiltIndexPath();
-    if (root == null || !Files.exists(root.resolve( StringUtil.toLowerCase(originalExtension.getName().getName())))) return null;
+  private static UncompressedZipFileSystem ourFs;
 
-    return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID)
-           ? (ProvidedIndexExtension<K, V>)new StubProvidedIndexExtension(root)
-           : new ProvidedIndexExtensionImpl<>(root, originalExtension);
+  @NotNull
+  @Override
+  public <K, V> Stream<ProvidedIndexExtension<K, V>> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
+    if (!originalExtension.dependsOnFileContent()) return Stream.empty();
+    Path root = getPrebuiltIndexPath();
+    if (root == null || !Files.exists(root)) return Stream.empty();
+
+    try {
+      // TODO properly close it
+      UncompressedZipFileSystem fs = getFs(root);
+
+      Path fsRoot = fs.getPath("/").getRoot();
+      return Files
+              .list(fsRoot)
+              .sorted(Comparator.comparing(p -> p.getFileName().toString()))
+              .map(p -> p.resolve(StringUtil.toLowerCase(originalExtension.getName().getName())))
+              .map(p -> {
+                return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID)
+                        ? (ProvidedIndexExtension<K, V>)new StubProvidedIndexExtension(p)
+                        : new ProvidedIndexExtensionImpl<>(p, originalExtension);
+              });
+    } catch (IOException e) {
+      LOG.error(e);
+      return Stream.empty();
+    }
+  }
+
+  private synchronized static UncompressedZipFileSystem getFs(@NotNull Path root) throws IOException {
+    if (ourFs == null) {
+      ourFs = new UncompressedZipFileSystem(root, new UncompressedZipFileSystemProvider());
+      ShutDownTracker.getInstance().registerShutdownTask(() -> {
+        try {
+          ourFs.close();
+        }
+        catch (IOException e) {
+          LOG.error(e);
+        }
+      });
+    }
+    return ourFs;
   }
 
   @Nullable
   private static Path getPrebuiltIndexPath() {
-    String path = System.getProperty(PREBUILT_INDEX_PATH_PROP);
+    String path = System.getProperty(PREBUILT_INDEX_ZIP_PROP);
     if (path == null) return null;
-    Path file;
-    try {
-      file = Paths.get(new URI(path));
-    }
-    catch (URISyntaxException e) {
-      LOG.error(e);
-      return null;
-    }
+    Path file = Paths.get(path);
     return Files.exists(file) ? file : null;
   }
 

platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java

@@ -52,6 +52,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.function.IntPredicate;
+import java.util.stream.Collectors;
 
 @State(name = "FileBasedIndex", storages = {
   @Storage(value = StoragePathMacros.CACHE_FILE),
@@ -196,14 +197,13 @@ public final class StubIndexImpl extends StubIndexEx implements PersistentStateC
         UpdatableIndex<K, Void, FileContent> index = new VfsAwareMapReduceIndex<>(wrappedExtension, memStorage, null, null, null, lock);
 
         if (stubUpdatingIndex instanceof MergedInvertedIndex) {
-          ProvidedIndexExtension<Integer, SerializedStubTree> ex =
-            ((MergedInvertedIndex<Integer, SerializedStubTree>)stubUpdatingIndex).getProvidedExtension();
-          if (ex instanceof StubProvidedIndexExtension) {
-            ProvidedIndexExtension<K, Void> providedStubIndexExtension =
-              ((StubProvidedIndexExtension)ex).findProvidedStubIndex(extension);
-            if (providedStubIndexExtension != null) {
-              index = ProvidedIndexExtension.wrapWithProvidedIndex(providedStubIndexExtension, wrappedExtension, index);
-            }
+          List<ProvidedIndexExtension<K, Void>> providedIndexExtensions = ((MergedInvertedIndex<Integer, SerializedStubTree>)stubUpdatingIndex)
+                  .getProvidedExtensions()
+                  .filter(ex -> ex instanceof StubProvidedIndexExtension)
+                  .map(ex -> ((StubProvidedIndexExtension)ex).findProvidedStubIndex(extension))
+                  .collect(Collectors.toList());
+          if (!providedIndexExtensions.isEmpty()) {
+            index = ProvidedIndexExtension.wrapWithProvidedIndex(providedIndexExtensions, wrappedExtension, index, ((MergedInvertedIndex<Integer, SerializedStubTree>)stubUpdatingIndex).getHashIndex());
           }
         }
 

platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java

@@ -37,7 +37,7 @@ public final class StubProcessingHelper extends StubProcessingHelperBase {
         return null;
       }
       SerializedStubTree tree = data.values().iterator().next();
-      StubIdList stubIdList = tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER, indexKey, key);
+      StubIdList stubIdList = tree.restoreIndexedStubs(indexKey, key);
       if (stubIdList == null) {
         LOG.error("Stub ids not found for key in index = " + indexKey.getName() + ", file type = " + file.getFileType());
         onInternalError(file);

platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java

@@ -37,7 +37,7 @@ class StubUpdatingForwardIndexAccessor implements ForwardIndexAccessor<Integer,
     Map<Integer, SerializedStubTree> data = dataRef.get();
     SerializedStubTree tree = ContainerUtil.isEmpty(data) ? null : ContainerUtil.getFirstItem(data.values());
     if (tree != null) {
-      tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER);
+      tree.restoreIndexedStubs();
     }
     return new StubCumulativeInputDiffBuilder(inputId, tree);
   }

platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java

@@ -47,6 +47,15 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
   public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");
 
   private static final FileBasedIndex.InputFilter INPUT_FILTER = file -> canHaveStub(file);
+  private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
+
+  public StubUpdatingIndex() {
+    myStubIndexesExternalizer = SerializedStubTree.IDE_USED_EXTERNALIZER;
+  }
+
+  public StubUpdatingIndex(StubForwardIndexExternalizer<?> stubIndexesExternalizer) {
+    myStubIndexesExternalizer = stubIndexesExternalizer;
+  }
 
   public static boolean canHaveStub(@NotNull VirtualFile file) {
     Project project = ProjectUtil.guessProjectForFile(file);
@@ -238,7 +247,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
   @NotNull
   @Override
   public DataExternalizer<SerializedStubTree> getValueExternalizer() {
-    return new SerializedStubTreeDataExternalizer();
+    return new SerializedStubTreeDataExternalizer(true, null, myStubIndexesExternalizer);
   }
 
   @NotNull

platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java

@@ -5,8 +5,6 @@ import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.psi.stubs.*;
-import com.intellij.util.indexing.FileBasedIndex;
-import com.intellij.util.indexing.FileBasedIndexImpl;
 import com.intellij.util.indexing.ID;
 import com.intellij.util.indexing.provided.ProvidedIndexExtension;
 import com.intellij.util.io.DataExternalizer;
@@ -16,7 +14,6 @@ import com.intellij.util.io.VoidDataExternalizer;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -52,7 +49,7 @@ public class StubProvidedIndexExtension implements ProvidedIndexExtension<Intege
       new SerializationManagerImpl(path.resolve(StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName())).resolve("rep.names"),
                                    true);
     Disposer.register(ApplicationManager.getApplication(), manager);
-    return new SerializedStubTreeDataExternalizer(false, manager);
+    return new SerializedStubTreeDataExternalizer(false, manager, StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE);
   }
 
   @Nullable

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java

@@ -93,6 +93,7 @@ import org.jetbrains.annotations.TestOnly;
 import java.io.*;
 import java.lang.ref.SoftReference;
 import java.lang.ref.WeakReference;
+import java.nio.file.Path;
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -449,9 +450,14 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
 
         UpdatableIndex<K, V, FileContent> index = createIndex(extension, new MemoryIndexStorage<>(storage, name));
 
-        ProvidedIndexExtension<K, V> providedExtension = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension);
-        if (providedExtension != null) {
-          index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtension, extension, index);
+        if (!(extension instanceof FileContentHashIndexExtension)) {
+          List<ProvidedIndexExtension<K, V>> providedExtensions = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension);
+          if (!providedExtensions.isEmpty()) {
+            Path[] paths = ContainerUtil.map2Array(providedExtensions, Path.class, ex -> ex.getIndexPath());
+            FileContentHashIndex contentHashIndex = ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(paths, state);
+
+            index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtensions, extension, index, contentHashIndex);
+          }
         }
 
         state.registerIndex(name,
@@ -2677,19 +2683,20 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
     }
   }
 
-  public synchronized FileContentHashIndex getFileContentHashIndex(@NotNull File enumeratorPath) {
-    UpdatableIndex<Integer, Void, FileContent> index = getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
+  public synchronized FileContentHashIndex getFileContentHashIndex(@Nullable Path[] enumeratorPaths, @NotNull IndexConfiguration state) {
+    UpdatableIndex<Long, Void, FileContent> index = state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
     if (index == null) {
+      LOG.assertTrue(enumeratorPaths != null);
       IndicesRegistrationResult registrationResult = new IndicesRegistrationResult();
       try {
-        registerIndexer(FileContentHashIndexExtension.create(enumeratorPath, ApplicationManager.getApplication()), myRegisteredIndexes.getState(), registrationResult);
+        registerIndexer(FileContentHashIndexExtension.create(enumeratorPaths, ApplicationManager.getApplication()), state, registrationResult);
         registrationResult.logChangedAndFullyBuiltIndices(LOG, "Version was changed for:", "Index is to be rebuilt:");
       }
       catch (IOException e) {
         throw new RuntimeException(e);
       }
     } else return (FileContentHashIndex)index;
-    return (FileContentHashIndex)getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
+    return (FileContentHashIndex)state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
   }
 
   private static final boolean INDICES_ARE_PSI_DEPENDENT_BY_DEFAULT = SystemProperties.getBooleanProperty("idea.indices.psi.dependent.default", true);

platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java

@@ -12,14 +12,14 @@ import java.util.concurrent.atomic.AtomicReference;
 /**
  * @author peter
  */
-enum RebuildStatus {
+public enum RebuildStatus {
   OK,
   REQUIRES_REBUILD,
   DOING_REBUILD;
 
   private static final Map<ID<?, ?>, AtomicReference<RebuildStatus>> ourRebuildStatus = new THashMap<>();
 
-  static void registerIndex(ID<?, ?> indexId) {
+  public static void registerIndex(ID<?, ?> indexId) {
     ourRebuildStatus.put(indexId, new AtomicReference<>(OK));
   }
 

platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java

@@ -119,7 +119,7 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
 
   @NotNull
   @Override
-  protected InputData<Key, Value> mapInput(@Nullable Input content) {
+  protected InputData<Key, Value> mapInput(int inputId, @Nullable Input content) {
     InputData<Key, Value> data;
     boolean containsSnapshotData = true;
     if (mySnapshotInputMappings != null && content != null) {
@@ -135,7 +135,7 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
         throw new RuntimeException(e);
       }
     }
-    data = super.mapInput(content);
+    data = super.mapInput(inputId, content);
     if (!containsSnapshotData && !UpdatableSnapshotInputMappingIndex.ignoreMappingIndexUpdate(content)) {
       try {
         return ((UpdatableSnapshotInputMappingIndex<Key, Value, Input>)mySnapshotInputMappings).putData(content, data);

platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java

@@ -13,8 +13,8 @@ import org.jetbrains.annotations.NotNull;
 import java.io.IOException;
 import java.util.Map;
 
-public class FileContentHashIndex extends VfsAwareMapReduceIndex<Integer, Void, FileContent> {
-  FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage<Integer, Void> storage) throws IOException {
+public class FileContentHashIndex extends VfsAwareMapReduceIndex<Long, Void, FileContent> {
+  FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage<Long, Void> storage) throws IOException {
     super(extension,
           storage,
           new PersistentMapBasedForwardIndex(IndexInfrastructure.getInputIndexStorageFile(extension.getName()).toPath(), false),
@@ -23,22 +23,22 @@ public class FileContentHashIndex extends VfsAwareMapReduceIndex<Integer, Void,
 
   @NotNull
   @Override
-  protected Computable<Boolean> createIndexUpdateComputation(@NotNull AbstractUpdateData<Integer, Void> updateData) {
+  protected Computable<Boolean> createIndexUpdateComputation(@NotNull AbstractUpdateData<Long, Void> updateData) {
     return new HashIndexUpdateComputable(super.createIndexUpdateComputation(updateData), updateData.newDataIsEmpty());
   }
 
-  public int getHashId(int fileId) throws StorageException {
-    Map<Integer, Void> data = getIndexedFileData(fileId);
-    if (data.isEmpty()) return 0;
+  public Long getHashId(int fileId) throws StorageException {
+    Map<Long, Void> data = getIndexedFileData(fileId);
+    if (data.isEmpty()) return FileContentHashIndexExtension.NULL_HASH_ID;
     return data.keySet().iterator().next();
   }
 
   @NotNull
-  IntIntFunction toHashIdToFileIdFunction() {
+  IntIntFunction toHashIdToFileIdFunction(int indexId) {
     return hash -> {
       try {
-        ValueContainer<Void> data = getData(hash);
-        assert data.size() == 1;
+        ValueContainer<Void> data = getData(FileContentHashIndexExtension.getHashId(hash, indexId));
+        if (data.size() == 0) return -1;
         return data.getValueIterator().getInputIdsIterator().next();
       }
       catch (StorageException e) {

platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java

@@ -4,51 +4,68 @@ package com.intellij.util.indexing.hash;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.Disposer;
+import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.ShutDownTracker;
 import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
+import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.*;
 import com.intellij.util.indexing.impl.IndexStorage;
-import com.intellij.util.io.*;
+import com.intellij.util.indexing.snapshot.IndexedHashesSupport;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.DataInputOutputUtil;
+import com.intellij.util.io.KeyDescriptor;
+import com.intellij.util.io.VoidDataExternalizer;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.DataInput;
 import java.io.DataOutput;
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
 
-public class FileContentHashIndexExtension extends FileBasedIndexExtension<Integer, Void> implements CustomImplementationFileBasedIndexExtension<Integer, Void>, CustomInputsIndexFileBasedIndexExtension<Integer>, Disposable {
+public class FileContentHashIndexExtension extends FileBasedIndexExtension<Long, Void> implements CustomImplementationFileBasedIndexExtension<Long, Void>, CustomInputsIndexFileBasedIndexExtension<Long>, Disposable {
   private static final Logger LOG = Logger.getInstance(FileContentHashIndexExtension.class);
-  public static final ID<Integer, Void> HASH_INDEX_ID = ID.create("file.content.hash.index");
+  public static final ID<Long, Void> HASH_INDEX_ID = ID.create("file.content.hash.index");
 
   @NotNull
-  private final ContentHashesUtil.HashEnumerator myEnumerator;
-  private final int myDirHash;
+  private final ContentHashesUtil.HashEnumerator[] myEnumerators;
 
   @NotNull
-  public static FileContentHashIndexExtension create(@NotNull File enumeratorDir, @NotNull Disposable parent) throws IOException {
-    FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDir);
+  public static FileContentHashIndexExtension create(@NotNull Path[] enumeratorDirs, @NotNull Disposable parent) throws IOException {
+    FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDirs);
+    RebuildStatus.registerIndex(extension.getName());
     Disposer.register(parent, extension);
     return extension;
   }
 
-  private FileContentHashIndexExtension(@NotNull File enumeratorDir) throws IOException {
-    myEnumerator = new ContentHashesUtil.HashEnumerator(enumeratorDir.toPath());
-    myDirHash = enumeratorDir.getAbsolutePath().hashCode();
+  private FileContentHashIndexExtension(@NotNull Path[] enumeratorDirs) throws IOException {
+    IOException[] exception = {null};
+    myEnumerators = ContainerUtil.map2Array(enumeratorDirs, ContentHashesUtil.HashEnumerator.class, d -> {
+      try {
+        return new ContentHashesUtil.HashEnumerator(d.getParent().resolve("hashes"));
+      }
+      catch (IOException e) {
+        exception[0] = e;
+        return null;
+      }
+    });
+    if (exception[0] != null) {
+      throw exception[0];
+    }
     ShutDownTracker.getInstance().registerShutdownTask(() -> closeEnumerator());
   }
 
   @NotNull
   @Override
-  public ID<Integer, Void> getName() {
+  public ID<Long, Void> getName() {
     return HASH_INDEX_ID;
   }
 
   @NotNull
   @Override
   public FileBasedIndex.InputFilter getInputFilter() {
-    throw new UnsupportedOperationException();
+    return file -> !file.isDirectory();
   }
 
   @Override
@@ -58,16 +75,15 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Integ
 
   @NotNull
   @Override
-  public DataIndexer<Integer, Void, FileContent> getIndexer() {
+  public DataIndexer<Long, Void, FileContent> getIndexer() {
     return fc -> {
-      byte[] hash = ((FileContentImpl)fc).getHash(false);
-      LOG.assertTrue(hash != null);
+      long hashId = getHashId(fc);
+      if (hashId != NULL_HASH_ID) return Collections.singletonMap(hashId, null);
+      byte[] hash = IndexedHashesSupport.getOrInitIndexedHash((FileContentImpl) fc, false);
       try {
-        int id;
-        synchronized (myEnumerator) {
-          id = myEnumerator.tryEnumerate(hash);
-        }
-        return id == 0 ? Collections.emptyMap() : Collections.singletonMap(id, null);
+        hashId = tryEnumerate(hash);
+        setHashId(fc, hashId);
+        return hashId == NULL_HASH_ID ? Collections.emptyMap() : Collections.singletonMap(hashId, null);
       }
       catch (IOException e) {
         throw new RuntimeException(e);
@@ -75,10 +91,44 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Integ
     };
   }
 
+  private Long tryEnumerate(byte[] hash) throws IOException {
+    for (int i = 0; i < myEnumerators.length; i++) {
+      ContentHashesUtil.HashEnumerator enumerator = myEnumerators[i];
+      //noinspection SynchronizationOnLocalVariableOrMethodParameter
+      synchronized (enumerator) {
+        int id = Math.abs(enumerator.tryEnumerate(hash));
+        if (id != 0) {
+          return getHashId(id, i);
+        }
+      }
+    }
+    return NULL_HASH_ID;
+  }
+
   @NotNull
   @Override
-  public KeyDescriptor<Integer> getKeyDescriptor() {
-    return EnumeratorIntegerDescriptor.INSTANCE;
+  public KeyDescriptor<Long> getKeyDescriptor() {
+    return new KeyDescriptor<Long>() {
+      @Override
+      public int getHashCode(Long value) {
+        return value.hashCode();
+      }
+
+      @Override
+      public boolean isEqual(Long val1, Long val2) {
+        return val1.longValue() == val2.longValue();
+      }
+
+      @Override
+      public void save(@NotNull DataOutput out, Long value) throws IOException {
+        DataInputOutputUtil.writeLONG(out, value);
+      }
+
+      @Override
+      public Long read(@NotNull DataInput in) throws IOException {
+        return DataInputOutputUtil.readLONG(in);
+      }
+    };
   }
 
   @NotNull
@@ -89,7 +139,7 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Integ
 
   @Override
   public int getVersion() {
-    return myDirHash;
+    return 0;
   }
 
   @Override
@@ -99,39 +149,66 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Integ
 
   @NotNull
   @Override
-  public DataExternalizer<Collection<Integer>> createExternalizer() {
-    return new DataExternalizer<Collection<Integer>>() {
+  public DataExternalizer<Collection<Long>> createExternalizer() {
+    return new DataExternalizer<Collection<Long>>() {
       @Override
-      public void save(@NotNull DataOutput out, Collection<Integer> value) throws IOException {
+      public void save(@NotNull DataOutput out, Collection<Long> value) throws IOException {
         assert value.isEmpty() || value.size() == 1;
-        DataInputOutputUtil.writeINT(out, value.isEmpty() ? 0 : value.iterator().next());
+        DataInputOutputUtil.writeLONG(out, value.isEmpty() ? 0 : value.iterator().next());
       }
 
       @Override
-      public Collection<Integer> read(@NotNull DataInput in) throws IOException {
-        int id = DataInputOutputUtil.readINT(in);
+      public Collection<Long> read(@NotNull DataInput in) throws IOException {
+        long id = DataInputOutputUtil.readLONG(in);
         return id == 0 ? Collections.emptyList() : Collections.singleton(id);
       }
     };
   }
 
   private void closeEnumerator() {
-    synchronized (myEnumerator) {
-      if (myEnumerator.isClosed()) return;
-      try {
-        myEnumerator.close();
-      }
-      catch (IOException e) {
-        LOG.error(e);
+    for (ContentHashesUtil.HashEnumerator enumerator : myEnumerators) {
+      synchronized (enumerator) {
+        if (enumerator.isClosed()) return;
+        try {
+          enumerator.close();
+        }
+        catch (IOException e) {
+          LOG.error(e);
+        }
       }
     }
   }
 
   @NotNull
   @Override
-  public UpdatableIndex<Integer, Void, FileContent> createIndexImplementation(@NotNull FileBasedIndexExtension<Integer, Void> extension,
-                                                                              @NotNull IndexStorage<Integer, Void> storage)
+  public UpdatableIndex<Long, Void, FileContent> createIndexImplementation(@NotNull FileBasedIndexExtension<Long, Void> extension,
+                                                                           @NotNull IndexStorage<Long, Void> storage)
     throws IOException {
     return new FileContentHashIndex(((FileContentHashIndexExtension)extension), storage);
   }
+
+  static int getIndexId(long hashId) {
+    return (int)(hashId >> 32);
+  }
+
+  static int getInternalHashId(long hashId) {
+    return (int)hashId;
+  }
+
+  static long getHashId(int internalHashId, int indexId) {
+    return (((long) indexId) << 32) | (internalHashId & 0xffffffffL);
+  }
+
+  public static final long NULL_HASH_ID = getHashId(0, -1);
+
+  public static final Key<Long> HASH_ID_KEY = Key.create("file.content.hash.id");
+
+  public static long getHashId(@NotNull FileContent content) {
+    Long value = HASH_ID_KEY.get(content);
+    return value == null ? NULL_HASH_ID : value;
+  }
+
+  public static void setHashId(@NotNull FileContent content, long hashId) {
+    HASH_ID_KEY.set(content, hashId);
+  }
 }

platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java

@@ -2,11 +2,13 @@
 package com.intellij.util.indexing.hash;
 
 import com.intellij.openapi.fileTypes.FileType;
+import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileVisitor;
 import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
+import com.intellij.psi.SingleRootFileViewProvider;
 import com.intellij.util.indexing.*;
 import com.intellij.util.indexing.impl.InputData;
 import com.intellij.util.indexing.impl.MapIndexStorage;
@@ -18,35 +20,33 @@ import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.*;
+import java.util.concurrent.atomic.LongAdder;
 
 public class HashBasedIndexGenerator<K, V> {
-  @NotNull private final File myOut;
-  @NotNull private final File myHashOut;
-  @NotNull private final FakeIndexExtension<K, V> myExtension;
-  @NotNull private final FileBasedIndex.InputFilter myInputFilter;
+  @NotNull
+  private final File myOut;
+  @NotNull
+  private final FakeIndexExtension<K, V> myExtension;
+  @NotNull
+  private final FileBasedIndex.InputFilter myInputFilter;
 
-  protected ContentHashesUtil.HashEnumerator myHashEnumerator;
   private InvertedIndex<K, V, FileContent> myIndex;
 
   public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension<K, V> indexExtension, @NotNull File out) {
     this(indexExtension.getKeyDescriptor(),
          indexExtension.getValueExternalizer(),
          indexExtension,
-         out,
-         out);
+         out
+    );
   }
 
   public HashBasedIndexGenerator(@NotNull KeyDescriptor<K> keyDescriptor,
                                  @NotNull DataExternalizer<V> valueExternalizer,
                                  @NotNull FileBasedIndexExtension<K, V> originalExtension,
-                                 @NotNull File out,
-                                 @NotNull File hashOut) {
+                                 @NotNull File out) {
     myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension);
     myOut = out;
-    myHashOut = hashOut;
 
     FileBasedIndex.InputFilter filter = originalExtension.getInputFilter();
 
@@ -65,8 +65,8 @@ public class HashBasedIndexGenerator<K, V> {
   }
 
   public void openIndex() throws IOException {
-    myHashEnumerator = getHashEnumerator();
     String indexName = myExtension.getName().getName();
+    boolean singleEntry = myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension;
     myIndex = new MapReduceIndex<K, V, FileContent>(myExtension, new MapIndexStorage<K, V>(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName).toPath(),
                                                                                            myExtension.getKeyDescriptor(),
                                                                                            myExtension.getValueExternalizer(),
@@ -77,6 +77,16 @@ public class HashBasedIndexGenerator<K, V> {
         //ignore
       }
     }, null, null) {
+      @NotNull
+      @Override
+      protected Map<K, V> mapByIndexer(int inputId, @NotNull FileContent content) {
+        Map<K, V> data = super.mapByIndexer(inputId, content);
+        if (singleEntry && !data.isEmpty()) {
+          data = Collections.singletonMap((K)(Integer)inputId, data.values().iterator().next());
+        }
+        return data;
+      }
+
       @Override
       protected void updateForwardIndex(int inputId, @NotNull InputData<K, V> data) throws IOException {
         super.updateForwardIndex(inputId, data);
@@ -95,60 +105,83 @@ public class HashBasedIndexGenerator<K, V> {
 
       @Override
       protected void requestRebuild(@NotNull Throwable e) {
-        throw new RuntimeException(e);
+        throw new RuntimeException("error while processing " + indexName, e);
       }
     };
   }
 
-  @NotNull
-  protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException {
-    return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes").toPath());
-  }
-
   protected void visitInputData(int hashId, @NotNull InputData<K, V> data) throws StorageException {
 
   }
 
   public void closeIndex() throws IOException {
     if (myIndex != null) myIndex.dispose();
-    if (myHashEnumerator != null) myHashEnumerator.close();
   }
 
+  public static void generate(@NotNull Collection<VirtualFile> roots,
+                              @NotNull Collection<HashBasedIndexGenerator<?, ?>> generators,
+                              @NotNull Project project,
+                              @NotNull File hashOut) {
 
-  public final void generate(@NotNull Collection<VirtualFile> roots) {
+    LongAdder l = new LongAdder();
     try {
-      openIndex();
+      ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(new File(hashOut, "hashes").toPath());
+
+      for (HashBasedIndexGenerator<?, ?> generator : generators) {
+        generator.openIndex();
+      }
 
       for (VirtualFile root : roots) {
         VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor<Boolean>() {
           @Override
           public boolean visitFile(@NotNull VirtualFile file) {
-            if (!file.isDirectory() && myInputFilter.acceptInput(file)) {
-              indexFile(file);
+
+            if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) {
+              for (HashBasedIndexGenerator<?, ?> generator : generators) {
+                if (generator.myInputFilter.acceptInput(file)) {
+                  l.increment();
+                  generator.indexFile(file, project, hashEnumerator);
+                }
+              }
             }
+
             return true;
           }
         });
       }
+
+      synchronized (hashEnumerator) {
+        hashEnumerator.close();
+      }
     }
     catch (IOException e) {
       throw new RuntimeException(e);
     }
     finally {
       try {
-        closeIndex();
+        for (HashBasedIndexGenerator<?, ?> generator : generators) {
+          generator.closeIndex();
+        }
       }
       catch (IOException e) {
         throw new RuntimeException(e);
       }
     }
+
+    System.out.println("Indexed " + l.sum() + " files to " + hashOut.getPath());
   }
 
-  protected void indexFile(@NotNull VirtualFile f) {
+  protected void indexFile(@NotNull VirtualFile f,
+                           @NotNull Project project,
+                           @NotNull ContentHashesUtil.HashEnumerator hashEnumerator) {
     try {
       FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray());
-      IndexedHashesSupport.initIndexedHash(fc);
-      int hashId = myHashEnumerator.enumerate(fc.getHash(false));
+      byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(fc, false);
+      int hashId;
+      synchronized (hashEnumerator) {
+        hashId = Math.abs(hashEnumerator.enumerate(hash));
+      }
+      fc.putUserData(IndexingDataKeys.PROJECT, project);
       if (!myIndex.update(hashId, fc).compute()) {
         throw new RuntimeException();
       }

platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java

@@ -2,14 +2,19 @@
 package com.intellij.util.indexing.hash;
 
 import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.IntIntFunction;
+import com.intellij.util.Processor;
 import com.intellij.util.indexing.*;
 import com.intellij.util.indexing.impl.IndexStorage;
 import com.intellij.util.indexing.impl.MapIndexStorage;
+import com.intellij.util.indexing.impl.UpdatableValueContainer;
 import com.intellij.util.indexing.provided.ProvidedIndexExtension;
+import com.intellij.util.io.PersistentEnumeratorBase;
+import com.intellij.util.io.PersistentHashMap;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 
@@ -19,17 +24,20 @@ class HashBasedMapReduceIndex<Key, Value> extends VfsAwareMapReduceIndex<Key, Va
 
   @NotNull
   static <Key, Value> HashBasedMapReduceIndex<Key, Value> create(@NotNull ProvidedIndexExtension<Key, Value> providedExtension,
-                                                                 @NotNull FileBasedIndexExtension<Key, Value> originalExtension)
+                                                                 @NotNull FileBasedIndexExtension<Key, Value> originalExtension,
+                                                                 @NotNull FileContentHashIndex hashIndex,
+                                                                 int providedIndexId)
     throws IOException {
     Path file = providedExtension.getIndexPath();
-    return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile()));
+    return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, hashIndex, providedIndexId);
   }
 
   private HashBasedMapReduceIndex(@NotNull Path baseFile,
                                   @NotNull FileBasedIndexExtension<Key, Value> originalExtension,
                                   @NotNull ProvidedIndexExtension<Key, Value> providedExtension,
-                                  @NotNull FileContentHashIndex hashIndex) throws IOException {
-    super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction()), null, null, null, null);
+                                  @NotNull FileContentHashIndex hashIndex,
+                                  int providedIndexId) throws IOException {
+    super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction(providedIndexId)), null, null, null, null);
     myProvidedExtension = providedExtension;
   }
 
@@ -42,19 +50,51 @@ class HashBasedMapReduceIndex<Key, Value> extends VfsAwareMapReduceIndex<Key, Va
                                                                      @NotNull FileBasedIndexExtension<Key, Value> originalExtension,
                                                                      @NotNull ProvidedIndexExtension<Key, Value> providedExtension,
                                                                      @NotNull IntIntFunction hashToFileId) throws IOException {
-    return new MapIndexStorage<Key, Value>(baseFile.resolve(originalExtension.getName().getName()),
-                                           providedExtension.createKeyDescriptor(),
-                                           providedExtension.createValueExternalizer(),
-                                           originalExtension.getCacheSize(),
-                                           originalExtension.keyIsUniqueForIndexedFile(),
-                                           true,
-                                           true,
-                                           hashToFileId) {
-      @Override
-      protected void checkCanceled() {
-        ProgressManager.checkCanceled();
-      }
-    };
+    return new MyMapIndexStorage<>(baseFile, originalExtension, providedExtension, hashToFileId);
   }
 
+  private static class MyMapIndexStorage<Key, Value>
+    extends MapIndexStorage<Key, Value>
+    implements VfsAwareIndexStorage<Key, Value> {
+    public MyMapIndexStorage(Path baseFile,
+                             FileBasedIndexExtension<Key, Value> originalExtension,
+                             ProvidedIndexExtension<Key, Value> providedExtension,
+                             IntIntFunction hashToFileId) throws IOException {
+      super(baseFile.resolve(originalExtension.getName().getName()), providedExtension.createKeyDescriptor(),
+            providedExtension.createValueExternalizer(), originalExtension.getCacheSize(), originalExtension.keyIsUniqueForIndexedFile(),
+            true, true, hashToFileId);
+    }
+
+    @Override
+    protected void checkCanceled() {
+      ProgressManager.checkCanceled();
+    }
+
+    @Override
+    public boolean processKeys(@NotNull Processor<? super Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter)
+      throws StorageException {
+      l.lock();
+      try {
+        myCache.clear(); // this will ensure that all new keys are made into the map
+        return doProcessKeys(processor);
+      }
+      catch (IOException e) {
+        throw new StorageException(e);
+      }
+      catch (RuntimeException e) {
+        return unwrapCauseAndRethrow(e);
+      }
+      finally {
+        l.unlock();
+      }
+    }
+
+    private boolean doProcessKeys(@NotNull Processor<? super Key> processor) throws IOException {
+      return myMap instanceof PersistentHashMap && PersistentEnumeratorBase.inlineKeyStorage(myKeyDescriptor)
+             // process keys and check that they're already present in map because we don't have separated key storage we must check keys
+             ? ((PersistentHashMap<Key, UpdatableValueContainer<Value>>)myMap).processKeysWithExistingMapping(processor)
+             // optimization: process all keys, some of them might be already deleted but we don't care. We just read key storage file here
+             : myMap.processKeys(processor);
+    }
+  }
 }

platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java

@@ -4,57 +4,74 @@ package com.intellij.util.indexing.hash;
 import com.intellij.openapi.util.Computable;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.util.Processor;
+import com.intellij.util.SmartList;
 import com.intellij.util.indexing.*;
 import com.intellij.util.indexing.impl.AbstractUpdateData;
+import com.intellij.util.indexing.impl.MergedValueContainer;
 import com.intellij.util.indexing.provided.ProvidedIndexExtension;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;
 
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Path;
+import java.nio.file.Files;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
+import java.util.stream.Stream;
 
 public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Value, FileContent> {
   @NotNull
-  private final HashBasedMapReduceIndex<Key, Value> myProvidedIndex;
+  private final HashBasedMapReduceIndex<Key, Value>[] myProvidedIndexes;
   @NotNull
   private final FileContentHashIndex myHashIndex;
   @NotNull
-  private final UpdatableIndex<Key, Value, FileContent> myBaseIndex;
+  public final UpdatableIndex<Key, Value, FileContent> myBaseIndex;
 
   @NotNull
-  public static <Key, Value> MergedInvertedIndex<Key, Value> create(@NotNull ProvidedIndexExtension<Key, Value> providedExtension,
+  public static <Key, Value> MergedInvertedIndex<Key, Value> create(@NotNull List<ProvidedIndexExtension<Key, Value>> providedExtensions,
                                                                     @NotNull FileBasedIndexExtension<Key, Value> originalExtension,
-                                                                    @NotNull UpdatableIndex<Key, Value, FileContent> baseIndex)
-    throws IOException {
-    Path file = providedExtension.getIndexPath();
-    HashBasedMapReduceIndex<Key, Value> index = HashBasedMapReduceIndex.create(providedExtension, originalExtension);
-    return new MergedInvertedIndex<>(index, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile()), baseIndex);
+                                                                    @NotNull UpdatableIndex<Key, Value, FileContent> baseIndex,
+                                                                    @NotNull FileContentHashIndex contentHashIndex) throws IOException {
+    HashBasedMapReduceIndex<Key, Value>[] providedIndexes = new HashBasedMapReduceIndex[providedExtensions.size()];
+    for (int i = 0; i < providedExtensions.size(); i++) {
+      ProvidedIndexExtension<Key, Value> extension = providedExtensions.get(i);
+      providedIndexes[i] = extension != null && Files.exists(extension.getIndexPath()) ? HashBasedMapReduceIndex.create(extension, originalExtension, contentHashIndex, i) : null;
+    }
+
+    return new MergedInvertedIndex<>(providedIndexes, contentHashIndex, baseIndex);
   }
 
-  public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex<Key, Value> index,
+  public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex<Key, Value>[] indexes,
                              @NotNull FileContentHashIndex hashIndex,
                              @NotNull UpdatableIndex<Key, Value, FileContent> baseIndex) {
-    myProvidedIndex = index;
+    myProvidedIndexes = indexes;
     myHashIndex = hashIndex;
     myBaseIndex = baseIndex;
   }
 
+
   @NotNull
-  public ProvidedIndexExtension<Key, Value> getProvidedExtension() {
-    return myProvidedIndex.getProvidedExtension();
+  public FileContentHashIndex getHashIndex() {
+    return myHashIndex;
+  }
+
+  @NotNull
+  public Stream<ProvidedIndexExtension<Key, Value>> getProvidedExtensions() {
+    return Stream.of(myProvidedIndexes).map(index -> index.getProvidedExtension());
   }
 
   @NotNull
   @Override
   public Computable<Boolean> update(int inputId, @Nullable FileContent content) {
     if (content != null) {
+      long hashId = FileContentHashIndexExtension.getHashId(content);
+      if (hashId != FileContentHashIndexExtension.NULL_HASH_ID) {
+        return () -> Boolean.TRUE;
+      }
       //TODO if content == null
       Computable<Boolean> update = myHashIndex.update(inputId, content);
       if (!((FileContentHashIndex.HashIndexUpdateComputable)update).isEmptyInput()) return update;
@@ -91,13 +108,28 @@ public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Valu
   @NotNull
   @Override
   public ValueContainer<Value> getData(@NotNull Key key) throws StorageException {
-    return MergedValueContainer.merge(myBaseIndex.getData(key), myProvidedIndex.getData(key));
+    List<ValueContainer<Value>> data = new SmartList<>();
+    data.add(myBaseIndex.getData(key));
+    for (HashBasedMapReduceIndex<Key, Value> index : myProvidedIndexes) {
+      if (index == null) continue;
+      data.add(index.getData(key));
+    }
+    return new MergedValueContainer<>(data);
   }
 
   @Override
   public boolean processAllKeys(@NotNull Processor<? super Key> processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter)
     throws StorageException {
-    return myBaseIndex.processAllKeys(processor, scope, idFilter) && myProvidedIndex.processAllKeys(processor, scope, idFilter);
+    if (!myBaseIndex.processAllKeys(processor, scope, idFilter)) {
+      return false;
+    }
+    for (HashBasedMapReduceIndex<Key, Value> index : myProvidedIndexes) {
+      if (index == null) continue;
+      if (!index.processAllKeys(processor, scope, idFilter)) {
+        return false;
+      }
+    }
+    return true;
   }
 
   @NotNull
@@ -123,9 +155,9 @@ public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Valu
   public Map<Key, Value> getIndexedFileData(int fileId) throws StorageException {
     Map<Key, Value> data = myBaseIndex.getIndexedFileData(fileId);
     if (!data.isEmpty()) return data;
-    int hashId = myHashIndex.getHashId(fileId);
-    if (hashId == 0) return Collections.emptyMap();
-    return myProvidedIndex.getIndexedFileData(hashId);
+    Long hashId = myHashIndex.getHashId(fileId);
+    if (hashId == null || hashId == FileContentHashIndexExtension.NULL_HASH_ID) return Collections.emptyMap();
+    return myProvidedIndexes[FileContentHashIndexExtension.getIndexId(hashId)].getIndexedFileData(FileContentHashIndexExtension.getInternalHashId(hashId));
   }
 
   @Override

platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java

@@ -1,65 +0,0 @@
-// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.intellij.util.indexing.hash;
-
-import com.intellij.util.indexing.ValueContainer;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-public class MergedValueContainer<Value> extends ValueContainer<Value> {
-  private final ValueContainer<Value> myContainer1;
-  private final ValueContainer<Value> myContainer2;
-
-  @NotNull
-  public static <Value> ValueContainer<Value> merge(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
-    if (container1.size() == 0) return container2;
-    if (container2.size() == 0) return container1;
-    return new MergedValueContainer<>(container1, container2);
-  }
-
-  private MergedValueContainer(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
-    myContainer1 = container1;
-    myContainer2 = container2;
-  }
-
-  @NotNull
-  @Override
-  public ValueIterator<Value> getValueIterator() {
-    return new ValueIterator<Value>() {
-      boolean mySecondIsUsed;
-      ValueIterator<Value> myCurrent = myContainer1.getValueIterator();
-
-      @NotNull
-      @Override
-      public IntIterator getInputIdsIterator() {
-        return myCurrent.getInputIdsIterator();
-      }
-
-      @Nullable
-      @Override
-      public IntPredicate getValueAssociationPredicate() {
-        return myCurrent.getValueAssociationPredicate();
-      }
-
-      @Override
-      public boolean hasNext() {
-        if (myCurrent.hasNext()) return true;
-        if (!mySecondIsUsed) {
-          myCurrent = myContainer2.getValueIterator();
-          mySecondIsUsed = true;
-          return hasNext();
-        }
-        return false;
-      }
-
-      @Override
-      public Value next() {
-        return myCurrent.next();
-      }
-    };
-  }
-
-  @Override
-  public int size() {
-    return myContainer1.size() + myContainer2.size();
-  }
-}

platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java

@@ -2,15 +2,19 @@
 package com.intellij.util.indexing.provided;
 
 import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.FileBasedIndexExtension;
+import com.intellij.util.indexing.FileContent;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.UpdatableIndex;
+import com.intellij.util.indexing.hash.FileContentHashIndex;
 import com.intellij.util.indexing.hash.MergedInvertedIndex;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.KeyDescriptor;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.List;
 
 public interface ProvidedIndexExtension<K, V> {
   Logger LOG = Logger.getInstance(ProvidedIndexExtension.class);
@@ -28,15 +32,16 @@ public interface ProvidedIndexExtension<K, V> {
   DataExternalizer<V> createValueExternalizer();
 
   @NotNull
-  static <K, V> UpdatableIndex<K, V, FileContent> wrapWithProvidedIndex(@NotNull ProvidedIndexExtension<K, V> providedIndexExtension,
+  static <K, V> UpdatableIndex<K, V, FileContent> wrapWithProvidedIndex(@NotNull List<ProvidedIndexExtension<K, V>> providedIndexExtensions,
                                                                         @NotNull FileBasedIndexExtension<K, V> originalExtension,
-                                                                        @NotNull UpdatableIndex<K, V, FileContent> index) {
+                                                                        @NotNull UpdatableIndex<K, V, FileContent> index,
+                                                                        @NotNull FileContentHashIndex contentHashIndex) {
     try {
-      return MergedInvertedIndex.create(providedIndexExtension, originalExtension, index);
+      return MergedInvertedIndex.create(providedIndexExtensions, originalExtension, index, contentHashIndex);
     }
     catch (IOException e) {
       LOG.error(e);
       return index;
     }
   }
-}
+}

platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java

@@ -4,18 +4,20 @@ package com.intellij.util.indexing.provided;
 import com.intellij.openapi.extensions.ExtensionPointName;
 import com.intellij.util.indexing.FileBasedIndexExtension;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
+import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public interface ProvidedIndexExtensionLocator {
   ExtensionPointName<ProvidedIndexExtensionLocator> EP_NAME = ExtensionPointName.create("com.intellij.fileBasedIndex.providedLocator");
 
-  @Nullable
-  <K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension);
+  @NotNull
+  <K, V> Stream<ProvidedIndexExtension<K, V>> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension);
 
-  @Nullable
-  static <K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
-    return EP_NAME.extensions().map(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).findFirst().orElse(null);
+  @NotNull
+  static <K, V> List<ProvidedIndexExtension<K, V>> findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
+    return EP_NAME.extensions().flatMap(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).collect(Collectors.toList());
   }
-}
+}

platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java

@@ -64,6 +64,17 @@ public class IndexedHashesSupport {
     content.setHashes(fileContentHash, documentHash != null ? documentHash : fileContentHash);
   }
 
+  @NotNull
+  public static byte[] getOrInitIndexedHash(@NotNull FileContentImpl content, boolean fromDocument) {
+    byte[] hash = content.getHash(fromDocument);
+    if (hash == null) {
+      initIndexedHash(content);
+      hash = content.getHash(fromDocument);
+      LOG.assertTrue(hash != null);
+    }
+    return hash;
+  }
+
   @NotNull
   private static byte[] calculateIndexedHashForFileContent(@NotNull FileContentImpl content, boolean binary) {
     byte[] contentHash = null;

platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java

@@ -280,13 +280,7 @@ public class SnapshotInputMappings<Key, Value, Input> implements UpdatableSnapsh
 
     Integer previouslyCalculatedContentHashId = content.getUserData(key);
     if (previouslyCalculatedContentHashId == null) {
-      byte[] hash = content.getHash(fromDocument);
-      if (hash == null) {
-        IndexedHashesSupport.initIndexedHash(content);
-        hash = content.getHash(fromDocument);
-        LOG.assertTrue(hash != null);
-      }
-
+      byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(content, fromDocument);
       previouslyCalculatedContentHashId = IndexedHashesSupport.enumerateHash(hash);
       content.putUserData(key, previouslyCalculatedContentHashId);
     }

platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java

@@ -1,10 +1,10 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package org.jetbrains.index.stubs;
 
+import com.google.common.collect.Maps;
 import com.intellij.openapi.application.PathManager;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
 import com.intellij.psi.stubs.*;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.indexing.FileBasedIndexExtension;
@@ -26,19 +26,19 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
   private final Set<StubIndexKey> myUsedKeys = new HashSet<>();
 
   public StubHashBasedIndexGenerator(@NotNull File out) {
-    super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out);
+    super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(
+            true,
+            null,
+            StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), getExtension(), out);
     for (StubIndexExtension<?, ?> stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) {
-      myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(),
-                                                                                             StubIndexImpl.StubIdExternalizer.INSTANCE,
-                                                                                             StubIndexImpl
-                                                                                               .wrapStubIndexExtension(stubIndexExtension),
-                                                                                             new File(out, getStubsDir()),
-                                                                                             out) {
-        @NotNull
-        @Override
-        protected ContentHashesUtil.HashEnumerator getHashEnumerator() {
-          return StubHashBasedIndexGenerator.this.myHashEnumerator;
-        }
+      FileBasedIndexExtension<?, Void> ex = StubIndexImpl
+              .wrapStubIndexExtension(stubIndexExtension);
+      myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(ex.getKeyDescriptor(),
+              ex.getValueExternalizer(),
+              ex,
+              new File(out, getStubsDir())
+      ) {
+
       });
     }
   }
@@ -59,7 +59,8 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
       Map<Object, StubIdList> value = entry.getValue();
       myUsedKeys.add(key);
       MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex();
-      index.updateWithMap(new UpdateData(hashId, value, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null));
+      Map<Object, Object> reducedValue = Maps.asMap(value.keySet(), k -> null);
+      index.updateWithMap(new UpdateData(hashId, reducedValue, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null));
     }
   }
 
@@ -83,6 +84,6 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
   }
 
   private static StubUpdatingIndex getExtension() {
-    return (StubUpdatingIndex)FileBasedIndexExtension.EXTENSION_POINT_NAME.extensions().filter(ex -> ex instanceof StubUpdatingIndex).findAny().get();
+    return (new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE));
   }
 }

platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java

@@ -264,7 +264,7 @@ public abstract class MapReduceIndex<Key,Value, Input> implements InvertedIndex<
 
   @NotNull
   protected UpdateData<Key, Value> calculateUpdateData(final int inputId, @Nullable Input content) {
-    final InputData<Key, Value> data = mapInput(content);
+    final InputData<Key, Value> data = mapInput(inputId, content);
     return createUpdateData(inputId,
                             data.getKeyValues(),
                             () -> getKeysDiffBuilder(inputId),
@@ -302,16 +302,21 @@ public abstract class MapReduceIndex<Key,Value, Input> implements InvertedIndex<
   }
 
   @NotNull
-  protected InputData<Key, Value> mapInput(@Nullable Input content) {
+  protected InputData<Key, Value> mapInput(int inputId, @Nullable Input content) {
     if (content == null) {
       return InputData.empty();
     }
-    Map<Key, Value> data = myIndexer.map(content);
+    Map<Key, Value> data = mapByIndexer(inputId,  content);
     checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer);
     checkCanceled();
     return new InputData<>(data);
   }
 
+  @NotNull
+  protected Map<Key, Value> mapByIndexer(int inputId, @NotNull Input content) {
+    return myIndexer.map(content);
+  }
+
   public abstract void checkCanceled();
 
   protected abstract void requestRebuild(@NotNull Throwable e);

platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java

@@ -12,13 +12,6 @@ public class MergedValueContainer<Value> extends ValueContainer<Value> {
   private final List<ValueContainer<Value>> myContainers;
   private int mySize;
 
-  @NotNull
-  public static <Value> ValueContainer<Value> merge(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
-    if (container1.size() == 0) return container2;
-    if (container2.size() == 0) return container1;
-    return new MergedValueContainer<>(Arrays.asList(container1, container2));
-  }
-
   public MergedValueContainer(@NotNull List<ValueContainer<Value>> containers) {
     if (containers.isEmpty()) {
       throw new IllegalArgumentException();