diff --git a/java/java-impl/src/META-INF/JavaPlugin.xml b/java/java-impl/src/META-INF/JavaPlugin.xml
index a218d100f9ea..815dca967245 100644
--- a/java/java-impl/src/META-INF/JavaPlugin.xml
+++ b/java/java-impl/src/META-INF/JavaPlugin.xml
@@ -1922,6 +1922,8 @@
+
+
diff --git a/java/java-impl/src/com/intellij/internal/DumpIndexAction.java b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java
new file mode 100644
index 000000000000..68e4ef8fd9b3
--- /dev/null
+++ b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java
@@ -0,0 +1,111 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal;
+
+import com.intellij.codeInspection.bytecodeAnalysis.BytecodeAnalysisIndex;
+import com.intellij.find.ngrams.TrigramIndex;
+import com.intellij.openapi.actionSystem.AnAction;
+import com.intellij.openapi.actionSystem.AnActionEvent;
+import com.intellij.openapi.fileChooser.FileChooser;
+import com.intellij.openapi.fileChooser.FileChooserDescriptor;
+import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory;
+import com.intellij.openapi.module.ModuleManager;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.roots.*;
+import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.vfs.VfsUtilCore;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileUrlChangeAdapter;
+import com.intellij.psi.impl.cache.impl.id.IdIndex;
+import com.intellij.psi.impl.java.stubs.index.JavaAutoModuleNameIndex;
+import com.intellij.psi.stubs.StubIndexKey;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.indexing.FileBasedIndexExtension;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.IndexableSetContributor;
+import com.intellij.util.indexing.hash.HashBasedIndexGenerator;
+import one.util.streamex.StreamEx;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.index.stubs.StubHashBasedIndexGenerator;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class DumpIndexAction extends AnAction {
+
+ @Override
+ public void actionPerformed(@NotNull AnActionEvent event) {
+ Project project = event.getProject();
+ if (project == null) return;
+ Stream libRoots = Arrays
+ .stream(ModuleManager.getInstance(project).getModules())
+ .flatMap(m -> Arrays.stream(ModuleRootManager.getInstance(m).getOrderEntries()))
+ .filter(e -> e instanceof LibraryOrSdkOrderEntry)
+ .flatMap(e -> Stream.concat(Arrays.stream(e.getFiles(OrderRootType.CLASSES)), Arrays.stream(e.getFiles(OrderRootType.SOURCES))));
+
+ Stream additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(),
+ IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream()));
+ Set roots = Stream.concat(libRoots, additionalRoots).collect(Collectors.toSet());
+
+ FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor();
+ descriptor.withTitle("Select Index Dump Directory");
+ VirtualFile file = FileChooser.chooseFile(descriptor, project, null);
+ if (file == null) return;
+ File out = VfsUtilCore.virtualToIoFile(file);
+ FileUtil.delete(out);
+ exportIndices(roots, out);
+ }
+
+ public static void exportIndices(@NotNull Set roots, @NotNull File out) {
+ StubHashBasedIndexGenerator generator = new StubHashBasedIndexGenerator(out);
+ generator.generate(roots);
+
+ getExportableIndices().forEach(ex -> {
+ HashBasedIndexGenerator indexGenerator = new HashBasedIndexGenerator(ex, out);
+ indexGenerator.generate(roots);
+ });
+ }
+
+ @NotNull
+ private static Stream getExportableIndices() {
+ //kt
+ Stream ktIndices =
+ FileBasedIndexExtension
+ .EXTENSION_POINT_NAME
+ .extensions()
+ .filter(id -> id.getName().getName().contains("kotlin"));
+
+ Set> xmlIndexIds = ContainerUtil.set(ID.findByName("XmlTagNames"),
+ ID.findByName("XmlNamespaces"),
+ ID.findByName("SchemaTypeInheritance"),
+ ID.findByName("DomFileIndex"),
+ ID.findByName("xmlProperties"));
+ //xml
+ Stream xmlIndices =
+ FileBasedIndexExtension
+ .EXTENSION_POINT_NAME
+ .extensions()
+ .filter(id -> xmlIndexIds.contains(id.getName()));
+
+ //base
+ Stream coreIndices =
+ FileBasedIndexExtension
+ .EXTENSION_POINT_NAME
+ .extensions()
+ .filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) ||
+ ex.getName().equals(IdIndex.NAME));
+
+ //java
+ Stream javaIndices =
+ FileBasedIndexExtension
+ .EXTENSION_POINT_NAME
+ .extensions()
+ .filter(ex -> ex instanceof BytecodeAnalysisIndex || ex instanceof JavaAutoModuleNameIndex);
+
+ return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices);
+ }
+}
diff --git a/platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java b/platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java
new file mode 100644
index 000000000000..9e8ceefbf022
--- /dev/null
+++ b/platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java
@@ -0,0 +1,82 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.index;
+
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.psi.stubs.StubUpdatingIndex;
+import com.intellij.psi.stubs.provided.StubProvidedIndexExtension;
+import com.intellij.util.indexing.FileBasedIndexExtension;
+import com.intellij.util.indexing.ID;
+import com.intellij.util.indexing.provided.ProvidedIndexExtension;
+import com.intellij.util.indexing.provided.ProvidedIndexExtensionLocator;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.KeyDescriptor;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+
+public class BasicProvidedExtensionLocator implements ProvidedIndexExtensionLocator {
+ private static final String PREBUILT_INDEX_PATH_PROP = "prebuilt.hash.index.dir";
+
+ @Nullable
+ @Override
+ public ProvidedIndexExtension findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension) {
+ File root = getPrebuiltIndexPath();
+ if (root == null || !new File(root, StringUtil.toLowerCase(originalExtension.getName().getName())).exists()) return null;
+
+ return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID)
+ ? (ProvidedIndexExtension)new StubProvidedIndexExtension(root)
+ : new ProvidedIndexExtensionImpl<>(root, originalExtension);
+ }
+
+ @Nullable
+ private static File getPrebuiltIndexPath() {
+ String path = System.getProperty(PREBUILT_INDEX_PATH_PROP);
+ if (path == null) return null;
+ File file = new File(path);
+ return file.exists() ? file : null;
+ }
+
+ private static class ProvidedIndexExtensionImpl implements ProvidedIndexExtension {
+ @NotNull
+ private final File myIndexFile;
+ @NotNull
+ private final ID myIndexId;
+ @NotNull
+ private final KeyDescriptor myKeyDescriptor;
+ @NotNull
+ private final DataExternalizer myValueExternalizer;
+
+ private ProvidedIndexExtensionImpl(@NotNull File file,
+ @NotNull FileBasedIndexExtension originalExtension) {
+ myIndexFile = file;
+ myIndexId = originalExtension.getName();
+ myKeyDescriptor = originalExtension.getKeyDescriptor();
+ myValueExternalizer = originalExtension.getValueExternalizer();
+ }
+
+ @NotNull
+ @Override
+ public File getIndexPath() {
+ return myIndexFile;
+ }
+
+ @NotNull
+ @Override
+ public ID getIndexId() {
+ return myIndexId;
+ }
+
+ @NotNull
+ @Override
+ public KeyDescriptor createKeyDescriptor() {
+ return myKeyDescriptor;
+ }
+
+ @NotNull
+ @Override
+ public DataExternalizer createValueExternalizer() {
+ return myValueExternalizer;
+ }
+ }
+}
diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java
new file mode 100644
index 000000000000..9cbf963a8259
--- /dev/null
+++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java
@@ -0,0 +1,230 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.hash;
+
+import com.intellij.openapi.fileTypes.FileType;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.VfsUtilCore;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileVisitor;
+import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
+import com.intellij.util.indexing.*;
+import com.intellij.util.indexing.impl.InputData;
+import com.intellij.util.indexing.impl.MapIndexStorage;
+import com.intellij.util.indexing.impl.MapReduceIndex;
+import com.intellij.util.io.DataExternalizer;
+import com.intellij.util.io.KeyDescriptor;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+public class HashBasedIndexGenerator {
+ @NotNull private final File myOut;
+ @NotNull private final File myHashOut;
+ @NotNull private final FakeIndexExtension myExtension;
+ @NotNull private final FileBasedIndex.InputFilter myInputFilter;
+
+ protected ContentHashesUtil.HashEnumerator myHashEnumerator;
+ private InvertedIndex myIndex;
+
+ public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension indexExtension, @NotNull File out) {
+ this(indexExtension.getKeyDescriptor(),
+ indexExtension.getValueExternalizer(),
+ indexExtension,
+ out,
+ out);
+ }
+
+ public HashBasedIndexGenerator(@NotNull KeyDescriptor keyDescriptor,
+ @NotNull DataExternalizer valueExternalizer,
+ @NotNull FileBasedIndexExtension originalExtension,
+ @NotNull File out,
+ @NotNull File hashOut) {
+ myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension);
+ myOut = out;
+ myHashOut = hashOut;
+
+ FileBasedIndex.InputFilter filter = originalExtension.getInputFilter();
+
+ if (filter instanceof FileBasedIndex.FileTypeSpecificInputFilter) {
+ Set fileTypes = new HashSet<>();
+ ((FileBasedIndex.FileTypeSpecificInputFilter)filter).registerFileTypesUsedForIndexing(fileTypes::add);
+ myInputFilter = file -> fileTypes.contains(file.getFileType()) && filter.acceptInput(file);
+ } else {
+ myInputFilter = filter;
+ }
+ }
+
+ @NotNull
+ public File getOut() {
+ return myOut;
+ }
+
+ public void openIndex() throws IOException {
+ myHashEnumerator = getHashEnumerator();
+ String indexName = myExtension.getName().getName();
+ myIndex = new MapReduceIndex(myExtension, new MapIndexStorage(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName),
+ myExtension.getKeyDescriptor(),
+ myExtension.getValueExternalizer(),
+ myExtension.getCacheSize(),
+ myExtension.keyIsUniqueForIndexedFile()) {
+ @Override
+ protected void checkCanceled() {
+ //ignore
+ }
+ }, null, null) {
+ @Override
+ protected void updateForwardIndex(int inputId, @NotNull InputData data) throws IOException {
+ super.updateForwardIndex(inputId, data);
+ try {
+ visitInputData(inputId, data);
+ }
+ catch (StorageException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void checkCanceled() {
+ //ignore
+ }
+
+ @Override
+ protected void requestRebuild(@NotNull Throwable e) {
+ throw new RuntimeException(e);
+ }
+ };
+ }
+
+ @NotNull
+ protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException {
+ return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes"));
+ }
+
+ protected void visitInputData(int hashId, @NotNull InputData data) throws StorageException {
+
+ }
+
+ public void closeIndex() throws IOException {
+ if (myIndex != null) myIndex.dispose();
+ if (myHashEnumerator != null) myHashEnumerator.close();
+ }
+
+
+ public final void generate(@NotNull Collection roots) {
+ try {
+ openIndex();
+
+ for (VirtualFile root : roots) {
+ VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor() {
+ @Override
+ public boolean visitFile(@NotNull VirtualFile file) {
+ if (!file.isDirectory() && myInputFilter.acceptInput(file)) {
+ indexFile(file);
+ }
+ return true;
+ }
+ });
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ finally {
+ try {
+ closeIndex();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ protected void indexFile(@NotNull VirtualFile f) {
+ try {
+ FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray());
+ byte[] hash = FileBasedIndexImpl.calculateHash(fc.getContent(), fc.getCharset(), f.getFileType(), fc.getFileType());
+ int hashId = myHashEnumerator.enumerate(hash);
+ if (!myIndex.update(hashId, fc).compute()) {
+ throw new RuntimeException();
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException("cant index " + f.getPath() + " for " + myExtension.getName().getName(), e);
+ }
+ }
+
+ public InvertedIndex getIndex() {
+ return myIndex;
+ }
+
+ private static final class FakeIndexExtension extends FileBasedIndexExtension {
+ @NotNull
+ private final KeyDescriptor myKeyDescriptor;
+ @NotNull
+ private final DataExternalizer myValueExternalizer;
+ @NotNull
+ private final FileBasedIndexExtension myOriginalExtension;
+
+ private FakeIndexExtension(@NotNull KeyDescriptor descriptor,
+ @NotNull DataExternalizer externalizer,
+ @NotNull FileBasedIndexExtension originalExtension) {
+ myKeyDescriptor = descriptor;
+ myValueExternalizer = externalizer;
+ myOriginalExtension = originalExtension;
+ }
+
+ @NotNull
+ @Override
+ public ID getName() {
+ return myOriginalExtension.getName();
+ }
+
+ @NotNull
+ @Override
+ public FileBasedIndex.InputFilter getInputFilter() {
+ return myOriginalExtension.getInputFilter();
+ }
+
+ @Override
+ public boolean dependsOnFileContent() {
+ return true;
+ }
+
+ @Override
+ public int getCacheSize() {
+ return myOriginalExtension.getCacheSize();
+ }
+
+ @Override
+ public boolean keyIsUniqueForIndexedFile() {
+ return myOriginalExtension.keyIsUniqueForIndexedFile();
+ }
+
+ @NotNull
+ @Override
+ public DataIndexer getIndexer() {
+ return myOriginalExtension.getIndexer();
+ }
+
+ @NotNull
+ @Override
+ public KeyDescriptor getKeyDescriptor() {
+ return myKeyDescriptor;
+ }
+
+ @NotNull
+ @Override
+ public DataExternalizer getValueExternalizer() {
+ return myValueExternalizer;
+ }
+
+ @Override
+ public int getVersion() {
+ return myOriginalExtension.getVersion();
+ }
+ }
+}
diff --git a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java
new file mode 100644
index 000000000000..74ceb672c0cb
--- /dev/null
+++ b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java
@@ -0,0 +1,88 @@
+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package org.jetbrains.index.stubs;
+
+import com.intellij.openapi.application.PathManager;
+import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
+import com.intellij.psi.stubs.*;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.indexing.FileBasedIndexExtension;
+import com.intellij.util.indexing.StorageException;
+import com.intellij.util.indexing.hash.HashBasedIndexGenerator;
+import com.intellij.util.indexing.impl.EmptyInputDataDiffBuilder;
+import com.intellij.util.indexing.impl.InputData;
+import com.intellij.util.indexing.impl.MapReduceIndex;
+import com.intellij.util.indexing.impl.UpdateData;
+import com.intellij.util.io.EnumeratorIntegerDescriptor;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator {
+ private final Map myStubIndexesGeneratorMap = new HashMap<>();
+ private final Set myUsedKeys = new HashSet<>();
+
+ public StubHashBasedIndexGenerator(@NotNull File out) {
+ super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out);
+ for (StubIndexExtension, ?> stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) {
+ myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(),
+ StubIndexImpl.StubIdExternalizer.INSTANCE,
+ StubIndexImpl
+ .wrapStubIndexExtension(stubIndexExtension),
+ new File(out, getStubsDir()),
+ out) {
+ @NotNull
+ @Override
+ protected ContentHashesUtil.HashEnumerator getHashEnumerator() {
+ return StubHashBasedIndexGenerator.this.myHashEnumerator;
+ }
+ });
+ }
+ }
+
+ @NotNull
+ private static String getStubsDir() {
+ return StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName());
+ }
+
+ @Override
+ protected void visitInputData(int hashId, @NotNull InputData data) throws StorageException {
+ super.visitInputData(hashId, data);
+ SerializedStubTree tree = ContainerUtil.getFirstItem(data.getKeyValues().values());
+ if (tree == null) return;
+ Map> map = tree.getStubIndicesValueMap();
+ for (Map.Entry> entry : map.entrySet()) {
+ StubIndexKey key = entry.getKey();
+ Map